diff --git a/.github/workflows/extended_checks.yml b/.github/workflows/extended_checks.yml index c70c2483fd..0c4b729fce 100644 --- a/.github/workflows/extended_checks.yml +++ b/.github/workflows/extended_checks.yml @@ -642,6 +642,10 @@ jobs: run: | set -eux PYTHONDONTWRITEBYTECODE=1 python3 examples/dasLLAMA/wasm/test_mint_models.py + # the Pocket converter's pure predicates (which tensor lands in which form) - numpy only, no + # torch; the runner's python carries no numpy, installed the way the news step installs markdown + python3 -m pip install numpy || python3 -m pip install --user numpy || python3 -m pip install --break-system-packages numpy + PYTHONDONTWRITEBYTECODE=1 python3 modules/dasLLAMA/harness/test_convert_pocket.py - name: "Test pr-babysit verdict core" if: matrix.role != 'modules' diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index bd01846ffa..0528be01e4 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -334,7 +334,8 @@ jobs: # 7. dasllama.io/examples — dasLLAMA itself compiled to wasm64, one app per example # (storyteller: stories15M writes, KittenTTS reads; storywish: you type the words, - # tinystories-instruct-27M writes). Each .das_package turns the GPU modules off, so the + # tinystories-instruct-27M writes, Pocket TTS reads; parrot: you talk, Pocket TTS clones the + # voice and reads the text you type in it). Each .das_package turns the GPU modules off, so the # host needs no Metal/Vulkan. Non-fatal like the games; the dasllama.io stage step # stages a card's page only when all three outputs exist. The models are prepared .dlim # images the stage step MINTS for this very build: examples/dasLLAMA/wasm/dlim_config is @@ -342,7 +343,7 @@ jobs: # bakes against the identity the browser build actually wants (a set minted for another # IMAGE_VERSION is declined by the program - the failure that put a black canvas on the # storyteller page for a day). - for ex in storyteller storywish; do + for ex in storyteller storywish parrot; do if ./bin/daslang utils/daspkg/main.das -- \ release wasm --root "examples/dasLLAMA/$ex" --out "$REPO/web/output64/examples"; then echo "$ex wasm build OK" @@ -639,7 +640,7 @@ jobs: rm -rf "_site_dasllama/examples/$1/models" printf '%s' "$1 - building

This example is being rebuilt and will be available shortly.

" > "_site_dasllama/examples/$1/$1.html" } - for ex in storyteller storywish; do + for ex in storyteller storywish parrot; do mkdir -p "_site_dasllama/examples/$ex" if [ -f "web/output64/examples/$ex/$ex.html" ] \ && [ -f "web/output64/examples/$ex/$ex.js" ] \ diff --git a/REVIEW_COMMON.md b/REVIEW_COMMON.md index f4c5f305ff..bd0cf372f8 100644 --- a/REVIEW_COMMON.md +++ b/REVIEW_COMMON.md @@ -135,5 +135,5 @@ obligation is a rule, and it lives in the flat list above. **Adding a rule starts with reading the whole checklist** - duplication, drift, and homeless placement all start with a rule appended by an author who had not just read the file. -**A rule the diff adds that is longer than every rule already in the file is split, its -exception dissolved, or its extra prose moved to the architecture doc.** +**A rule the diff adds or lengthens that is longer than every other rule in the file is split, +or its extra prose moved to the architecture doc.** diff --git a/doc/source/reference/tutorials/dasLLAMA_16_text_to_speech.rst b/doc/source/reference/tutorials/dasLLAMA_16_text_to_speech.rst index 173a16fed1..84be1da32c 100644 --- a/doc/source/reference/tutorials/dasLLAMA_16_text_to_speech.rst +++ b/doc/source/reference/tutorials/dasLLAMA_16_text_to_speech.rst @@ -247,12 +247,13 @@ Cloning a voice =============== ``caps().cloning`` says whether the model takes a voice from a recording. -Pocket TTS does: a few seconds of one speaker, mono, at the model's own rate, -become a voice in the roster. ``load_audio_mono`` decodes a wav, flac, mp3 or -ogg file to that rate, and ``tts_register_voice`` adds the samples under the -name you give. From then on the name works like any bundled voice. A clip -longer than a minute is refused, and a phoneme model panics here: it has no -voice to take. +A Pocket TTS file with its codec encoder does: a few seconds of one speaker, +mono, at the model's own rate, become a voice in the roster. ``load_audio_mono`` +decodes a wav, flac, mp3 or ogg file to that rate, and ``tts_register_voice`` +adds the samples under the name you give. From then on the name works like any +bundled voice. A clip longer than a minute is refused; a Pocket file converted +without the encoder (a one-voice file for a page) reports ``cloning`` false and +refuses by name; a phoneme model panics here: it has no voice to take. .. code-block:: das @@ -262,15 +263,18 @@ voice to take. let mine <- synthesize(m, "daslang speaks in my voice.", "me") } -The two weight lanes -==================== +The weight lanes +================ The decoder and generator matrix multiplies are served from one of two prepared images beside the GGUF. The q8 lane holds those weights as Q8_0 quants and is what a load serves by default. The f32 lane holds the file's own planes; it is the reference the parity tests hold the q8 lane against. A published Pocket file already holds Q8_0 weights, so its q8 lane reads them as -they are and its f32 lane dequantizes them. +they are and its f32 lane dequantizes them. The small Pocket files hold Q4_K +planes for the backbone and the codec transformers too: an unpinned load serves +those planes as they are through the engine's K-quant kernels, a third lane, +while a pin to q8 or f32 requantizes or dequantizes them at load. ``tts_serves_q8`` answers which lane the next load takes. ``set_tts_q8`` pins it, and ``reset_tts_q8`` returns to the default. The pin is process-wide diff --git a/examples/dasLLAMA/ARCHITECTURE.md b/examples/dasLLAMA/ARCHITECTURE.md index 403aff73bd..11599c7fea 100644 --- a/examples/dasLLAMA/ARCHITECTURE.md +++ b/examples/dasLLAMA/ARCHITECTURE.md @@ -10,9 +10,16 @@ checklist is `REVIEW.md` beside this file. The engine these programs drive is do frame, KittenTTS reads each finished sentence. `main.das` is the whole program, `web_shell.html` the page around its canvas, `.das_package` the release, `models.json` its model set. - `storywish/` - a browser example: the typed words become a request in the TinyStoriesInstruct - corpus's layout, tinystories-instruct-27M writes the story, KittenTTS reads it. Same four files; + corpus's layout, tinystories-instruct-27M writes the story, Pocket TTS reads it in one baked + voice from a file without the codec encoder (text in, no packs, no cloning). Same four files; `wish.das` holds the request side pure (typed line -> words -> prompt, the field-line stop) so a test reaches it without a window. +- `parrot/` - a browser example: you press record and talk, Silero VAD ends the take when you go + quiet, Pocket TTS clones the voice from the take (a file with its codec encoder and its + roster, which speaks until a take replaces it), and the text in the box is read aloud in it on + the say button; recording again + replaces the voice. Same four files. Nothing leaves the program: the take is cloned in memory + and never written. - `wasm/dlim_config/` - a wasm-only program: prints the running build's DlimConfiguration JSON. `wasm/mint_models.py` - the deploy's staging step for a browser example's model set. `wasm/run_node.js` - runs the wasm64 engine host under node. @@ -25,8 +32,8 @@ checklist is `REVIEW.md` beside this file. The engine these programs drive is do wasm` builds it to wasm64 for dasllama.io from the same `main.das` the desktop run uses. The checklist's rules about the browser build bind browser examples and nothing else. - **A witness line** is a line a browser example logs under its own name (`storyteller: ...`, - `storywish: ...`); a smoke test under `modules/dasLLAMA/tests/` matches such lines as - substrings, so their words and order are an interface. + `storywish: ...`, `parrot: ...`); a smoke test under `modules/dasLLAMA/tests/` matches such + lines as substrings, so their words and order are an interface. ## 3. Mechanisms @@ -37,19 +44,25 @@ the desktop and `requestAnimationFrame` in the browser. The `.das_package` disab for the wasm build, so the host needs no Metal or Vulkan, and names the shell that fetches the models and starts the program on a click (audio needs the gesture). The desktop run and the page therefore exercise the same code, which is why a browser-only failure is a language-runtime fact -worth a rule rather than an app bug. +worth a rule rather than an app bug. A page the browser restores from its back-forward cache +(Back to another page, then Forward) comes back with the program's workers and the audio output +frozen mid-frame and out of step, and the first sound is whatever the output ring held; the +shell reloads such a page (`pageshow` with `persisted`), so it starts from the gate again. ### 3.2 The speech thread and its stream {#speech-thread-stream} Speech runs on its own thread so the frame loop never blocks on synthesis. The frame thread -pushes sentences into a stream as archived `Line` records and pops finished clips from a second -stream; a `SeqBox` carries the number of the story being told, so a queued sentence of a story -the user replaced is skipped instead of synthesized. The thread's own setup - the TTS model path -and the voice - rides the same sentence stream ahead of the first sentence. A string captured by -the thread's lambda would be a pointer into the frame thread's heap, which that thread reuses on -its own schedule; a browser worker starts slowly enough to read story text where the path was. -An archived message is copied out of the stream into the reader's heap, so the stream is the one -channel that is safe for a string. +pushes its requests into a stream as archived records (a sentence for the story examples; for +parrot a text to say or a take to clone, the PCM riding in the record) and pops finished clips +from a second stream; a `SeqBox` carries the number of the story (parrot: the say) being told, so +a queued sentence of one the user replaced is skipped instead of synthesized. The thread's own +setup - the TTS model path and the voice - rides the same request stream ahead of the first +request. Parrot's thread answers a say with its chunk count before the first clip, so the frame +thread can tell the last clip from a pause. A string captured by the thread's lambda would be a +pointer into the frame thread's heap, which that thread reuses on its own schedule; a browser +worker starts slowly enough to read story text where the path was. An archived message is copied +out of the stream into the reader's heap, so the stream is the one channel that is safe for a +string. ### 3.3 Input is polled {#polled-keys} @@ -57,14 +70,26 @@ A browser example reads the keyboard with `glfwGetKey` each frame, edge-detected never through a GLFW callback. In the browser build a callback lambda fires from a JavaScript event outside any frame of the program, where the example's state is not live, and the program traps. A printable GLFW key code is its upper-case ASCII, so the key range doubles as the -character range for a typed line, and repeats come from a hold timer. +character range for a typed line, and repeats come from a hold timer. The mouse is read the same +way: parrot's buttons are text, and a click is `glfwGetMouseButton` edge-detected against the +label's own box (the glyph quads rise above the pen position) in design pixels. In the browser +the surface is the document viewport, and the page's stage sits below the nav, so the picture is +letterboxed; Emscripten maps a click through the canvas element's box with one ratio per axis, +which is exact only when that box is the picture - so a shell sizes the canvas element to the +letterboxed box (`max-width`/`max-height` on the replaced element) instead of stretching it over +the stage with `object-fit`. ### 3.4 The model set is minted for the build that ships it Each browser example's `models.json` names its source files by Hugging Face repository, file -and sha256. `wasm/mint_models.py` fetches them (cached by sha256), bakes each GGUF into a -`.dlim` image against the wasm64 build's own DlimConfiguration, copies the packs, writes -`models/manifest.json` (the file list, their sizes, the IMAGE_VERSION the images carry) and +and sha256, in three lists: `images` (a GGUF the build bakes into a `.dlim`), `packs` (a +front-end pack) and `files` (a GGUF that is its own served form - a Pocket TTS file); a fourth +list, `tree`, names a file the repository itself carries by its repo-relative path and sha256 +(the voice-activity weights). `wasm/mint_models.py` +fetches the published ones (cached by sha256), bakes each image against the wasm64 build's own +DlimConfiguration, copies the packs, files and tree files as they are, writes +`models/manifest.json` (the file list, their sizes, the IMAGE_VERSION the images carry - a set +with no image carries the version the deploy expects) and stamps that version into the page's `/* @image-version */ 0` slot. The shell reads the manifest, refuses a set minted for another version before fetching it, and shows a program abort's last engine lines on the page. The configuration the mint bakes against comes from the wasm build @@ -77,6 +102,22 @@ is keyed by the build's identity and a set minted for a previous build is declin Its `.das_package` disables the GPU tiers exactly as the browser examples' do, so the configuration it prints is the one their programs run with. +### 3.6 Parrot's take {#the-take} + +The microphone is opened at the speech model's own rate (the speech thread reports it, with +whether the file clones, before the first take), mono, and drained on the frame thread every +frame into the take; the take is resampled to 16 kHz by linear interpolation with one running +position across drains, so the stream the Silero iterator hears has no seam where the drains +met, and the iterator is the only reader of that rate. The take ends on the stop button, two +seconds after the iterator's last speech end, at the model's 60 s clip cap, or when the device +has delivered nothing for six seconds (a refused microphone opens but never delivers); the ring's +tail is drained before the device stops, since stopping frees the ring. The clip is the speech +plus a quarter second at each end, never longer than the cap, and goes to the speech thread as a +clone request, so the clone runs off the frame thread like a synthesis. A take with no speech in +it is dropped, and the status says whether the device gave nothing, silence, or too little. A +take starts by cancelling a say in flight - a clip still playing would be recorded - and the +pure side of all this (`take.das`) is what the model-free cells test. + ## 4. Exception ledger None. diff --git a/examples/dasLLAMA/REVIEW.md b/examples/dasLLAMA/REVIEW.md index f9f212081f..bb0ffb29a5 100644 --- a/examples/dasLLAMA/REVIEW.md +++ b/examples/dasLLAMA/REVIEW.md @@ -6,16 +6,26 @@ A browser example is a subfolder here with a `web_shell.html` (`ARCHITECTURE.md` sec. 2); the rules below bind browser examples. -**Never install a GLFW callback - any `glfwSet*Callback` - in a browser example; poll each key -with `glfwGetKey` every frame and edge-detect it.** In the browser build a callback fires outside -any frame of the program and the program traps (`ARCHITECTURE.md` sec. 3.3). +**Never install a GLFW callback - any `glfwSet*Callback` - in a browser example; poll each input +every frame and edge-detect it - `glfwGetKey` for keys, `glfwGetMouseButton` for buttons.** In the +browser build a callback fires outside any frame of the program and the program traps +(`ARCHITECTURE.md` sec. 3.3). + +**A browser example's `web_shell.html` gives the canvas element `max-width` / `max-height` so the +element's box is exactly the rendered image - never `object-fit` on a canvas stretched to fill the +page area around it.** A click maps through the element's box with one ratio per axis, so a +stretched box mis-maps every click (`ARCHITECTURE.md` sec. 3.3). + +**A browser example's `web_shell.html` must reload a page the browser restored from its +back-forward cache - a `pageshow` handler that reloads when `persisted` is set.** Such a page +comes back with its workers and audio output frozen out of step (`ARCHITECTURE.md` sec. 3.1). **A diff that changes or drops a witness line - a line a browser example logs under its own name - updates every test under `modules/dasLLAMA/tests/` that matches it, in the same change.** The smoke tests match witness lines as substrings, so the words and their order are an interface (`ARCHITECTURE.md` sec. 2). -**A diff that adds a model file to a browser example's `models.json` names it by the repository -it is published in and its sha256, never by a local path or a branch name.** The deploy fetches -the file by that name and refuses one whose hash moved; a local path stages nothing on the -runner (`ARCHITECTURE.md` sec. 3.4). +**A diff that adds a model file to a browser example's `models.json` names its sha256 and a +location that cannot move - a Hugging Face repository, or a repo-relative path in this repository +under `tree` - never a machine-local path or a branch name.** The deploy fetches by that name and +refuses a file whose hash moved (`ARCHITECTURE.md` sec. 3.4). diff --git a/examples/dasLLAMA/parrot/.das_package b/examples/dasLLAMA/parrot/.das_package new file mode 100644 index 0000000000..40b18a980b --- /dev/null +++ b/examples/dasLLAMA/parrot/.das_package @@ -0,0 +1,19 @@ +options gen2 + +require daslib/daspkg + +[export] +def package() { + package_name("parrot") + package_description("Parrot: talk for a few seconds, Pocket TTS clones the voice, and the text you type is read aloud in it - dasGlfw + dasOpenGL + dasAudio over dasLLAMA") +} + +[export] +def release() { + release_main("main.das") // one source for the desktop run AND wasm64 (daspkg release wasm) + release_web_shell("web_shell.html") // fetches the models into MEMFS, then a click starts the program (audio needs the gesture) + // host-only GPU tiers: absent from the wasm build, so their guarded requires resolve as absent + release_wasm_disable_module("dasvulkan") + release_wasm_disable_module("dasmetal") + release_wasm_disable_module("dasaccelerate") +} diff --git a/examples/dasLLAMA/parrot/main.das b/examples/dasLLAMA/parrot/main.das new file mode 100644 index 0000000000..0f89be2370 --- /dev/null +++ b/examples/dasLLAMA/parrot/main.das @@ -0,0 +1,743 @@ +options gen2 +options persistent_heap +options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) + +require dasllama/dasllama // the facade: the TTS model, its caps, the clone verb, the chunker, the synthesis, and Silero (when a take has speech and when it stops) +require take // the pure side: the take's numbers, the text box's rules, a button's box +require daslib/jobque_boost +require daslib/strings_boost +require daslib/clargs +require daslib/fio +require daslib/archive +require glfw/glfw_boost +require live/glfw_live +require opengl/opengl_boost +require opengl/opengl_cache +require opengl/opengl_ttf +require audio/audio_boost +require audio/audio_record +require live_host +require math +require daslib/math_boost // ortho_rh / compose for the text layer +require strings + +// Parrot. You press record and talk for as long as you like; the take ends when you press stop +// or when you have been quiet for two seconds. Pocket TTS clones the voice from the take, and +// the text in the box - a poem to begin with, or whatever you type - is read aloud in it when +// you press say. Record again and the voice is replaced. Escape quits. +// +// bin/daslang -jit examples/dasLLAMA/parrot/main.das -- --models [--text "..."] [--clip voice.wav] +// +// holds the speech model - a Pocket TTS file with its codec encoder, which is what clones - +// and silero_vad.bin. The recording stays in the program: nothing leaves the machine, and in the +// browser build nothing leaves the tab. --clip clones from a file instead of the microphone (the +// smoke rail). The microphone is drained on the frame thread; the clone and the synthesis run on +// the speech thread, fed through one stream and answering through another, and the frame thread +// plays the clips back to back. + +[CommandLineArgs] +struct ParrotArgs { + @clarg_short = "m" + @clarg_doc = "Directory holding the speech model and the voice-activity model (default: the current directory)" + models : string = "." + + @clarg_doc = "The TTS model file inside --models (a Pocket TTS file with its codec encoder)" + tts_model : string = "pocket-tts-en-kq.gguf" + + @clarg_doc = "The voice-activity model file inside --models (default: the tree's own modules/dasLLAMA/models/silero_vad.bin)" + vad_model : string + + @clarg_short = "t" + @clarg_doc = "The text in the box at the start (default: a poem)" + text : string + + @clarg_doc = "Clone the voice from this clip instead of the microphone (wav, flac, mp3 or ogg)" + clip : string + + @clarg_doc = "Clone from --clip, say the text, quit when it has been read out - the smoke rail" + smoke : bool + + @clarg_doc = "Stop after this many frames (0 = never) - the smoke rail" + max_frames : int + + @clarg_short = "?" + @clarg_name = "show-help" + @clarg_doc = "Show this help and exit" + help : bool +} + +let POEM = "Whose woods these are I think I know.\nHis house is in the village though;\nHe will not see me stopping here\nTo watch his woods fill up with snow.\n\nMy little horse must think it queer\nTo stop without a farmhouse near\nBetween the woods and frozen lake\nThe darkest evening of the year.\n\nHe gives his harness bells a shake\nTo ask if there is some mistake.\nThe only other sound's the sweep\nOf easy wind and downy flake.\n\nThe woods are lovely, dark and deep,\nBut I have promises to keep,\nAnd miles to go before I sleep,\nAnd miles to go before I sleep." + +let TEXT_SIZE = 0.66 +let SMALL_SIZE = 0.5 +let VOICE_NAME = "you" +let READ_OUT_STATUS = "read out - change the text and say again, or record a new voice" + +enum Phase { + idle + recording + cloning +} + +//! frame thread -> speech thread: the setup, a clip to clone, a text to say, or the stop +struct Ask { + kind : int //! 0 = the model path, 1 = a clone, 2 = a text, 3 = stop + text : string + gen : int //! which say it belongs to: a chunk of a say the user replaced is skipped + pcm : array + rate : int +} + +//! speech thread -> frame thread: the model's facts, a voice cloned, a clip of a say, or how many clips a say will have +struct Answer { + kind : int //! 0 = ready (the rate, cloning in gen, the roster size in text), 1 = cloned (seconds in text), 2 = a clip, 3 = the say's chunk count (in text) + text : string + gen : int + pcm : array + rate : int +} + +var g_args = ParrotArgs() +var g_phase = Phase.idle +var g_lines : array //! the text box, one line each +var g_status = "" +var g_voice_ready = false +var g_voice_seconds = 0.0 +var g_says = 0 +var g_chunks_owed = 0 //! clips of the current say still to play; the speech thread names the count first +var g_count_known = false +var g_read_out_say = 0 //! the last say whose read-out was logged, so the line is written once +var g_take : array //! the microphone as recorded, 24 kHz mono +var g_take_scratch : array +var g_vad_model = VadModel() +var g_vad = VadIter() +var g_ears = VadResampler() +var g_ears_out : array +var g_speech_seen = false +var g_in_speech = false +var g_speech_start24 = 0l +var g_speech_end24 = 0l +var g_quiet_since_s = 0.0 +var g_take_started_s = 0.0 +var g_last_frames_s = 0.0 //! when the device last delivered a frame; a device that never does ends the take +var g_frames_seen = false +var g_level = 0.0 +var g_level_peak = 0.0 +var g_ready = false //! the speech thread has the model: its rate, whether it clones, its roster +var g_mic_rate = MIC_RATE +var g_can_clone = false +var g_roster = 0 +var g_pending_clip : array //! a --clip decoded before the model was ready + +var g_ask : Stream? +var g_answer : Stream? +var g_say_now : SeqBox? +var g_speech_done : Channel? +var g_clips : array +var g_playing_sid = INVALID_SID +var g_elapsed_s = 0.0 +var g_speaking_until = 0.0 +var g_audio_initialized = false +var g_asch : AudioSystemChannels +var g_font : Font? +var g_frames = 0 +var g_key_was : bool[512] // GLFW key codes end at GLFW_KEY_MENU = 348 +var g_mouse_was = false +var g_backspace_held_s = 0.0 +var display_w = 0 +var display_h = 0 + +// ===== the speech thread ===== + +//! the speech thread's setup rides its own stream ahead of the asks: the model path +[arch(at = "../ARCHITECTURE.md#speech-thread-stream")] +def start_speech_thread(var ask, answer : Stream?; var now : SeqBox?; var done : Channel?) { + new_thread() <| @capture(= ask, = answer, = now, = done) { + setup_dasllama_jobque() // the fork-context pool is per context: without it every parallel kernel clones the program + var tts_path = "" + ask |> pop_archive() $(var a : Ask&) { + tts_path = clone_string(a.text) + } + var inscope m <- load_tts_model(tts_path) + var inscope c <- caps(m) + var voice = empty(c.voices) ? "" : c.voices[0] + var ready = Answer(kind = 0, text = "{length(c.voices)}", gen = c.cloning ? 1 : 0, rate = c.sample_rate) + answer |> push_archive(ready) + var running = true + while (running) { + ask |> pop_archive() $(var a : Ask&) { + if (a.kind == 3) { + running = false + } elif (a.kind == 1) { + tts_register_voice(m, VOICE_NAME, a.pcm, a.rate) + voice = VOICE_NAME + var cloned = Answer(kind = 1, text = "{float(length(a.pcm)) / float(a.rate)}", gen = a.gen, rate = a.rate) + answer |> push_archive(cloned) + } elif (a.kind == 2) { + var inscope chunks <- tts_chunks(m, a.text) + var count = Answer(kind = 3, text = "{length(chunks)}", gen = a.gen) + answer |> push_archive(count) + for (chunk in chunks) { + var current = a.gen + now |> read() $(gen : int) { + current = gen + } + break if (a.gen < current) // a say the user replaced: its clips would be dropped unheard + var inscope s <- synthesize(m, chunk, voice) + var clip = Answer(kind = 2, gen = a.gen, pcm := s.pcm, rate = s.sample_rate) + answer |> push_archive(clip) + } + } + } + } + ask |> release() + answer |> release() + now |> seq_box_release() + done |> notify_and_release() + } +} + +def poll_answers() { + g_answer |> try_pop() $(bytes) { + var a : Answer + mem_archive_load(bytes, a) + if (a.kind == 0) { + g_ready = true + g_mic_rate = a.rate + g_can_clone = a.gen != 0 + g_roster = to_int(a.text) + g_status = g_can_clone ? "press record and talk, then say" : "this file cannot clone - parrot needs a Pocket file with its codec encoder; say reads in the model's voice" + if (!empty(g_pending_clip)) { + if (!g_can_clone) { + panic("parrot: --clip needs a file that clones; {g_args.tts_model} carries no codec encoder") + } + clone_clip(g_pending_clip, g_mic_rate) + } + } elif (a.kind == 1) { + g_voice_ready = true + g_voice_seconds = to_float(a.text) + g_phase = Phase.idle + g_status = "your voice is ready - press say" + //! the smoke rail's witness lines, here and below: modules/dasLLAMA/tests/test_parrot.das reads them word for word + to_log(LOG_INFO, "parrot: cloned {a.text} s of speech\n") + if (g_args.smoke) { + say_text() + } + } elif (a.gen == g_says) { + if (a.kind == 3) { + g_chunks_owed = to_int(a.text) + g_count_known = true + } else { + g_clips |> emplace(a) + } + } + } + return if (g_elapsed_s < g_speaking_until) + g_playing_sid = INVALID_SID + if (!empty(g_clips)) { + let seconds = float(length(g_clips[0].pcm)) / float(max(g_clips[0].rate, 1)) + var pcm <- g_clips[0].pcm + g_playing_sid = play_sound_from_pcm(g_clips[0].rate, 1, pcm) + g_clips |> erase(0) + g_speaking_until = g_elapsed_s + seconds + g_chunks_owed-- + } elif (g_count_known && g_chunks_owed <= 0 && g_says > g_read_out_say) { + g_read_out_say = g_says + if (g_phase == Phase.idle) { + g_status = READ_OUT_STATUS + } + to_log(LOG_INFO, "parrot: say {g_says} is read out\n") + } +} + +// ===== the take ===== + +def start_take() { + if (g_phase == Phase.cloning) { + g_status = "cloning your voice - a moment, then record again" + return + } + return if (g_phase != Phase.idle) + if (!g_ready) { + g_status = "the model is still loading" + return + } + if (!g_can_clone) { + g_status = "this file cannot clone - parrot needs a Pocket file with its codec encoder" + return + } + cancel_say() // a clip still playing would be recorded + if (!sound_record_start(g_mic_rate, 1, g_mic_rate * 4, -1)) { + g_status = "no microphone - is one connected, and allowed?" + return + } + g_take |> clear() + g_take |> reserve(int(TAKE_CAP_S) * g_mic_rate) + g_take_scratch |> resize(g_mic_rate) + vad_iter_reset(g_vad, default_vad_opts()) + g_ears.pos = 0.0lf + g_speech_seen = false + g_in_speech = false + g_speech_start24 = 0l + g_speech_end24 = 0l + g_quiet_since_s = g_elapsed_s + g_take_started_s = g_elapsed_s + g_last_frames_s = g_elapsed_s + g_frames_seen = false + g_level = 0.0 + g_level_peak = 0.0 + g_phase = Phase.recording + g_status = "listening - talk for as long as you like, then press stop or go quiet" + to_log(LOG_INFO, "parrot: recording\n") +} + +//! the frames the device has since delivered, onto the take +def append_frames(n : int) { + var ssq = 0.0 + let base = length(g_take) + g_take |> ensure_capacity(base + n) + g_take |> resize(base + n) + for (i in range(n)) { + let v = g_take_scratch[i] + g_take[base + i] = v + ssq += v * v + } + let rms = sqrt(ssq / float(n)) + g_level = max(rms, g_level * 0.85) + g_level_peak = max(g_level_peak, rms) + g_frames_seen = true + g_last_frames_s = g_elapsed_s +} + +[arch(at = "../ARCHITECTURE.md#the-take")] +def drain_take() { + let n = sound_record_read(g_take_scratch) + if (n > 0) { + append_frames(n) + resample_step(g_ears, g_take, g_ears_out) + vad_iter_feed(g_vad_model, g_vad, g_ears_out) $(ev) { + let at24 = vad_to_mic(ev.sample) + if (ev.kind == VadEventKind.speech_start) { + if (!g_speech_seen) { + g_speech_start24 = at24 + } + g_speech_seen = true + g_in_speech = true + } else { + g_speech_end24 = at24 + g_in_speech = false + g_quiet_since_s = g_elapsed_s + } + } + } + let seconds = float(length(g_take)) / float(g_mic_rate) + if (take_ends(g_speech_seen, g_in_speech, g_elapsed_s - g_quiet_since_s, seconds, g_elapsed_s - g_last_frames_s)) { + stop_take() + } +} + +def stop_take() { + return if (g_phase != Phase.recording) + drain_rest() // before the device stops: stopping frees the ring + sound_record_stop() + let dropped = sound_record_overflow_frames() + if (dropped > 0l) { + to_log(LOG_WARNING, "parrot: {dropped} frames were dropped while the frame loop stalled - the take is spliced there\n") + } + if (!g_speech_seen) { + g_phase = Phase.idle + let seconds = float(length(g_take)) / float(g_mic_rate) + g_status = (!g_frames_seen + ? "nothing heard - the microphone gave no audio at all: it was refused, or there is none" + : g_level_peak < 0.001 + ? "nothing heard - the microphone gave silence for {seconds} s: is it allowed for this page, and on?" + : "nothing heard - the microphone peaked at {g_level_peak} over {seconds} s, too quiet for speech: come closer and talk") + to_log(LOG_INFO, "parrot: nothing heard - peak {g_level_peak} over {seconds} s\n") + return + } + let w = clip_window(g_speech_start24, g_speech_end24, long_length(g_take), g_in_speech) + var clip : array + clip |> resize(w.to - w.from) + for (i in range64(w.to - w.from)) { + clip[i] = g_take[w.from + i] + } + clone_clip(clip, g_mic_rate) +} + +//! the ring's tail before the device stops +def drain_rest() { + for (_i in range(8)) { + let n = sound_record_read(g_take_scratch) + break if (n <= 0) + append_frames(n) + } +} + +def clone_clip(var clip : array; rate : int) { + g_phase = Phase.cloning + g_status = "cloning your voice from {float(length(clip)) / float(rate)} seconds..." + var a = Ask(kind = 1, gen = g_says, pcm <- clip, rate = rate) + g_ask |> push_archive(a) +} + +//! a say in flight is dropped: its clip stops, its queued clips go unplayed, and the speech thread +//! skips its remaining chunks at the next chunk boundary +def cancel_say() { + if (g_playing_sid != INVALID_SID) { + stop(g_playing_sid, 0.05) + g_playing_sid = INVALID_SID + } + g_says++ + g_say_now |> publish(g_says) + g_clips |> clear() + g_speaking_until = g_elapsed_s + g_chunks_owed = 0 + g_count_known = false +} + +// ===== saying ===== + +def say_text() { + let text = strip(join(g_lines, "\n")) + return if (empty(text) || g_phase != Phase.idle || !g_ready) // a say while recording would be recorded + if (!g_voice_ready && g_roster == 0) { + g_status = "no voice to read in - record one" + return + } + cancel_say() + var a = Ask(kind = 2, text = text, gen = g_says) + g_ask |> push_archive(a) + g_status = g_voice_ready ? "reading in your voice..." : "reading in the model's own voice - record to hear yours" + to_log(LOG_INFO, "parrot: say {g_says} in voice {g_voice_ready ? VOICE_NAME : "default"}, {length(text)} characters\n") +} + +// ===== the text box ===== + +def set_text(text : string) { + delete g_lines + g_lines <- lines_of(text) +} + +//! down this frame and not the last +def key_pressed_now(key : int) : bool { + let down = glfwGetKey(live_window, key) == GLFW_PRESS + let was = g_key_was[key] + g_key_was[key] = down + return down && !was +} + +def key_down(key : int) : bool { + return glfwGetKey(live_window, key) == GLFW_PRESS +} + +//! Ctrl or Command: the say chord works the same on every desktop +def ctrl_down() : bool { + return (key_down(GLFW_KEY_LEFT_CONTROL) || key_down(GLFW_KEY_RIGHT_CONTROL) || key_down(GLFW_KEY_LEFT_SUPER) || key_down(GLFW_KEY_RIGHT_SUPER)) +} + +def shift_down() : bool { + return key_down(GLFW_KEY_LEFT_SHIFT) || key_down(GLFW_KEY_RIGHT_SHIFT) +} + +//! a printable GLFW key code is its upper-case ASCII, so the key range IS the character range +[arch(at = "../ARCHITECTURE.md#polled-keys")] +def poll_keys() { + let shift = shift_down() + for (key in range(GLFW_KEY_SPACE, GLFW_KEY_GRAVE_ACCENT + 1)) { + if (key_pressed_now(key)) { + let c = typed_char(key, shift) + if (c >= 0) { + append_char(g_lines, c) + } + } + } + if (key_pressed_now(GLFW_KEY_BACKSPACE)) { + erase_last(g_lines) + g_backspace_held_s = 0.0 + } elif (glfwGetKey(live_window, GLFW_KEY_BACKSPACE) == GLFW_PRESS) { + g_backspace_held_s += get_dt() + if (g_backspace_held_s > 0.4) { // held: a repeat every 60 ms after the first 400 + erase_last(g_lines) + g_backspace_held_s = 0.34 + } + } + if (key_pressed_now(GLFW_KEY_ENTER) || key_pressed_now(GLFW_KEY_KP_ENTER)) { + if (ctrl_down()) { + say_text() + } else { + new_line(g_lines) + } + } + if (key_pressed_now(GLFW_KEY_TAB)) { + toggle_record() + } +} + +def toggle_record() { + if (g_phase == Phase.recording) { + stop_take() + } else { + start_take() + } +} + +// ===== the screen ===== + +//! the text layer's coordinates are design pixels against a 1280x720 reference; the smaller of the two ratios +//! governs, so a window narrower than 16:9 shrinks the text instead of cutting the lines on the right +def hud_scale() : float { + let fit = min(float(display_w) / 1280.0, float(display_h) / 720.0) + return max(fit, 0.5) +} + +def design_height() : float { + return float(display_h) / hud_scale() +} + +def text_mvp(x, y, scale : float) : float4x4 { + let projection = ortho_rh(0.0, float(display_w), float(display_h), 0.0, -1.0, 1.0) + let model = compose(float3(x, y, 0.0), float4(0.0, 0.0, 0.0, 1.0), float3(scale, scale, 1.0)) + return projection * model +} + +def draw_text(text : string; x, y : float; size : float; tint : float3) { + return if (g_font == null || empty(text)) + var quads <- (*g_font) |> create_quads(text) + let s = hud_scale() * size + (*g_font) |> draw_quads(quads, text_mvp(x * hud_scale(), y * hud_scale(), s), tint) + delete quads +} + +//! a text's box in design pixels at `size`, relative to the pen it is drawn at: left, top, right, bottom +//! (the glyphs sit above the baseline, so the top is negative) +def text_box(text : string; size : float) : float4 { + return float4(0.0) if (g_font == null || empty(text)) + var quads <- (*g_font) |> create_quads(text) + let d = quads_dim(quads) + delete quads + return float4(d.vmin.x, d.vmin.y, d.vmax.x, d.vmax.y) * size +} + +//! a button: its label in a bracket, drawn at a pen position; it answers to the label's own box +struct Button { + text : string + x : float + y : float + box : float4 //! the label's box relative to the pen: left, top, right, bottom +} + +def button(caption : string; x, y : float) : Button { + return Button(text = caption, x = x, y = y, box = text_box("[ {caption} ]", TEXT_SIZE)) +} + +//! the pen position after the button, with a gap +def after(b : Button) : float { + return b.x + b.box.z + 30.0 +} + +def draw_button(b : Button; lit : bool) { + draw_text("[ {b.text} ]", b.x, b.y, TEXT_SIZE, lit ? float3(0.98, 0.75, 0.35) : float3(0.75, 0.72, 0.66)) +} + +def meter_text() : string { + let cells = 24 + let lit = min(cells, int(g_level * 40.0 * float(cells))) + return build_string() $(var w) { + for (i in range(cells)) { + w |> write(i < lit ? "#" : ".") + } + } +} + +def draw_screen() { + let ink = float3(0.93, 0.9, 0.82) + let dim = float3(0.55, 0.52, 0.48) + let accent = float3(0.98, 0.75, 0.35) + draw_text("parrot", 60.0, 60.0, 0.6, dim) + draw_text("your recording stays in this window: nothing is uploaded, and the voice is gone when you close it", 60.0, 96.0, SMALL_SIZE, dim) + var y = 150.0 + let step = 30.0 + let bottom = design_height() - 150.0 + let visible = max(1, int((bottom - y) / step)) + let first = max(0, length(g_lines) - visible) + for (i in range(first, length(g_lines))) { + let caret = i == length(g_lines) - 1 && (g_frames / 30) % 2 == 0 ? "_" : "" + draw_text("{g_lines[i]}{caret}", 60.0, y, TEXT_SIZE, ink) + y += step + } + let row = design_height() - 120.0 + let rec = button(g_phase == Phase.recording ? "stop" : "record", 60.0, row) + draw_button(rec, g_phase == Phase.recording) + var x = after(rec) + if (g_phase == Phase.recording) { + let meter = meter_text() + draw_text(meter, x, row, TEXT_SIZE, accent) + draw_text("{float(length(g_take)) / float(MIC_RATE)} s", x + text_box(meter, TEXT_SIZE).z + 20.0, row, TEXT_SIZE, dim) + } else { + let sayb = button("say", x, row) + draw_button(sayb, true) + x = after(sayb) + draw_text(g_voice_ready ? "voice: yours, from {g_voice_seconds} s" : "voice: the model's own until you record", x, row, SMALL_SIZE, dim) + } + draw_text(g_status, 60.0, design_height() - 70.0, SMALL_SIZE, g_phase == Phase.idle ? accent : dim) + draw_text("Tab records and stops Ctrl+Enter says Enter is a new line", 60.0, design_height() - 40.0, SMALL_SIZE, dim) +} + +//! the two buttons answer a click: the cursor in design pixels against their rectangles +def poll_mouse() { + let down = glfwGetMouseButton(live_window, GLFW_MOUSE_BUTTON_1) == GLFW_PRESS + let pressed = down && !g_mouse_was + g_mouse_was = down + return if (!pressed) + var win_w, win_h : int + glfwGetWindowSize(live_window, unsafe(addr(win_w)), unsafe(addr(win_h))) + let c = glfwGetCursorPos(live_window) * float(display_w) / float(max(win_w, 1)) //! the cursor comes in window points; the text layer is laid out in framebuffer pixels, twice that on a retina screen + let scale = hud_scale() + let px = c.x / scale + let py = c.y / scale + let row = design_height() - 120.0 + let rec = button(g_phase == Phase.recording ? "stop" : "record", 60.0, row) + if (hit(rec, px, py)) { + toggle_record() + return + } + if (g_phase != Phase.recording) { + let sayb = button("say", after(rec), row) + if (hit(sayb, px, py)) { + say_text() + } + } +} + +//! inside the label's box, with a margin around it +def hit(b : Button; px, py : float) : bool { + return inside_box(b.box, b.x, b.y, px, py, 8.0) +} + +// ===== the program ===== + +[export] +def init() { + var inscope r <- parse_args(type) + if (r |> is_err) { + panic("parrot: {r |> unwrap_err}") + } + g_args <- r |> move_unwrap + set_text(empty(g_args.text) ? POEM : g_args.text) + g_status = "press record and talk, then say" + if (!empty(g_args.clip)) { + // decoded before the window and the thread exist, so a clip that is not audio refuses here and the program ends + delete g_pending_clip + g_pending_clip <- load_audio_mono(g_args.clip, MIC_RATE) // cloned once the thread says the file can; the model's rate is 24 kHz + if (empty(g_pending_clip)) { + panic("parrot: {g_args.clip} did not decode") + } + let cap = int(TAKE_CAP_S) * MIC_RATE + if (length(g_pending_clip) > cap) { + to_log(LOG_INFO, "parrot: {g_args.clip} is longer than the {int(TAKE_CAP_S)} s a voice is cloned from - the first {int(TAKE_CAP_S)} s are taken\n") + g_pending_clip |> resize(cap) + } + to_log(LOG_INFO, "parrot: cloning from {g_args.clip}\n") + g_status = "cloning from {g_args.clip}..." + } + live_create_window("Parrot", 1280, 720) + cache_ttf_objects() + g_font = cache_font("{get_das_root()}/modules/dasStbImage/fonts/droidsansmono.ttf") + if (!g_audio_initialized) { + g_asch = audio_system_create() + g_audio_initialized = true + } + create_job_que() + setup_dasllama_jobque() + delete g_vad_model + g_vad_model <- load_vad_model(empty(g_args.vad_model) ? "{get_das_root()}/modules/dasLLAMA/models/silero_vad.bin" : path_join(g_args.models, g_args.vad_model)) + g_ask = unsafe(stream_create()) + g_answer = unsafe(stream_create()) + g_say_now = seq_box_create() + g_say_now |> publish(g_says) + g_speech_done = unsafe(channel_create()) + g_speech_done |> append(1) + var setup = Ask(kind = 0, text = path_join(g_args.models, g_args.tts_model)) + g_ask |> push_archive(setup) + start_speech_thread(g_ask, g_answer, g_say_now, g_speech_done) +} + +[export] +def update() { + if (!live_begin_frame()) { + return + } + g_frames++ + g_elapsed_s += get_dt() + live_get_framebuffer_size(display_w, display_h) + glViewport(0, 0, display_w, display_h) + glClearColor(0.07, 0.06, 0.09, 1.0) + glClear(GL_COLOR_BUFFER_BIT) + glDisable(GL_DEPTH_TEST) + glEnable(GL_BLEND) + glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA) + + if (glfwGetKey(live_window, GLFW_KEY_ESCAPE) == GLFW_PRESS) { + glfwSetWindowShouldClose(live_window, 1) + } + poll_keys() + poll_mouse() + if (g_phase == Phase.recording) { + drain_take() + } + poll_answers() + draw_screen() + live_end_frame() +} + +[export] +def shutdown() { + if (g_phase == Phase.recording) { + sound_record_stop() + } + cancel_say() // the thread leaves a say in flight at its next chunk instead of finishing it + var stop_ask = Ask(kind = 3) + g_ask |> push_archive(stop_ask) + g_speech_done |> join() + g_say_now |> seq_box_release() + unsafe { + channel_remove(g_speech_done) + stream_remove(g_ask) + stream_remove(g_answer) + } + if (g_audio_initialized) { + audio_system_finalize(g_asch.command, g_asch.next_sid) + g_audio_initialized = false + } + delete g_vad_model + destroy_job_que() + live_destroy_window() +} + +def read_out() : bool { + return g_says > 0 && g_count_known && g_chunks_owed <= 0 && empty(g_clips) && g_elapsed_s >= g_speaking_until +} + +def done_for_smoke() : bool { + return true if (g_args.max_frames > 0 && g_frames >= g_args.max_frames) + return g_args.smoke && read_out() +} + +// eval_main_loop drives the block once per frame: a blocking while-loop natively, the +// browser's requestAnimationFrame on the web - one main for both +[export] +def main() { + init() + eval_main_loop() { + update() + return false if (done_for_smoke()) + return !exit_requested() + } + let finished = read_out() + shutdown() + if (g_args.smoke) { + to_log(LOG_INFO, finished + ? "parrot: the text was read out after {g_frames} frames\n" + : "parrot: the frame cap stopped the run after {g_frames} frames\n") + } +} diff --git a/examples/dasLLAMA/parrot/models.json b/examples/dasLLAMA/parrot/models.json new file mode 100644 index 0000000000..0ffa920d7e --- /dev/null +++ b/examples/dasLLAMA/parrot/models.json @@ -0,0 +1,8 @@ +{ + "files": [ + { "file": "pocket-tts-en-kq.gguf", "repo": "borisbat/dasllama-tts", "sha256": "2475a1ed8d49eb72c9d9b8c38f10f91ef5b03c7cd6e9fe43fdf7ab00ae1a0a25" } + ], + "tree": [ + { "file": "silero_vad.bin", "path": "modules/dasLLAMA/models/silero_vad.bin", "sha256": "33d2121d08c033eeb08f73ce5b6130c02d2e84c34547f8e9f97aeb408fc5a82a" } + ] +} diff --git a/examples/dasLLAMA/parrot/take.das b/examples/dasLLAMA/parrot/take.das new file mode 100644 index 0000000000..cf28806b3c --- /dev/null +++ b/examples/dasLLAMA/parrot/take.das @@ -0,0 +1,116 @@ +options gen2 + +require math +require strings +require daslib/strings_boost + +// The pure side of parrot: what the take does with the numbers the microphone and the voice +// detector hand it, the text box's editing rules, and a button's hit box - none of it touches +// a device, a thread or the screen, so modules/dasLLAMA/tests/test_parrot.das holds it +// without a window or a model. main.das drives the devices and the speech thread. + +let MIC_RATE = 24000 //! the model's own rate: the take is cloned as recorded +let VAD_RATE = 16000 //! Silero listens at 16 kHz; the take is resampled for it alone +let SILENCE_ENDS_TAKE_S = 2.0 //! quiet this long after speech ends the take +let TAKE_CAP_S = 60.0 //! the model's clip cap: the take ends here, and the clip never exceeds it +let TAKE_PAD_S = 0.25 //! kept around the speech at both ends +let NO_FRAMES_ENDS_TAKE_S = 6.0 //! a device that delivers nothing this long ends the take: the microphone was refused, or is not there +let WRAP_CHARS = 74 //! droidsansmono is monospace: characters are the wrap unit +let MAX_LINES = 24 + +//! 24 kHz to 16 kHz by linear interpolation over the whole take: the next position to sample +//! carries across drains, so the stream Silero hears has no seam where the drains met +struct VadResampler { + pos : double //! the next 24 kHz position to sample +} + +//! the 16 kHz samples the take has grown enough to yield since the last call +def resample_step(var r : VadResampler; take : array; var out : array) { + out |> clear() + let n = long_length(take) + let step = double(MIC_RATE) / double(VAD_RATE) + while (int64(r.pos) + 1l < n) { + let j = int64(r.pos) + let f = float(r.pos - double(j)) + out |> push(take[j] * (1.0 - f) + take[j + 1l] * f) + r.pos += step + } +} + +//! a 16 kHz sample index of the detector as a 24 kHz index into the take +def vad_to_mic(sample16 : int64) : int64 { + return sample16 * int64(MIC_RATE) / int64(VAD_RATE) +} + +//! whether the take ends now: speech was heard and the quiet after it has lasted, the cap, or a +//! device that gave nothing for long enough to be gone +def take_ends(speech_seen, in_speech : bool; quiet_for_s, seconds, silent_device_for_s : float) : bool { + return (speech_seen && !in_speech && quiet_for_s >= SILENCE_ENDS_TAKE_S) || seconds >= TAKE_CAP_S || silent_device_for_s >= NO_FRAMES_ENDS_TAKE_S +} + +//! the clip's window in the take: the speech plus a pad at both ends, an open speech running to +//! the take's end, and never longer than the model's clip cap +def clip_window(start24, end24, total : int64; in_speech : bool) : tuple { + let pad = int64(TAKE_PAD_S * float(MIC_RATE)) + let cap = int64(TAKE_CAP_S) * int64(MIC_RATE) + let from = clamp(start24 - pad, 0l, total) + var to = min(total, (in_speech ? total : end24) + pad) + to = clamp(to, from, from + cap) + return (from = from, to = to) +} + +//! the character a key lands in the box as: letters follow shift, the list's punctuation and the +//! shifted digits that spell it, spaces; -1 for a key the box does not take +def typed_char(key : int; shift : bool) : int { + return -1 if (key < 0 || key >= 128) + var c = key + if (is_alpha(c)) { + c = shift ? c : c + 32 // GLFW hands the upper-case code + } elif (shift) { + c = c == '1' ? '!' : c == '/' ? '?' : c == ';' ? ':' : c == '\'' ? '"' : c == '9' ? '(' : c == '0' ? ')' : c == '-' ? '_' : c + } + let ok = is_alpha(c) || is_number(c) || c == ' ' || c == ',' || c == '.' || c == '\'' || c == '"' || c == '-' || c == ';' || c == ':' || c == '!' || c == '?' || c == '(' || c == ')' || c == '_' + return ok ? c : -1 +} + +//! a character onto the last line, unless that line is full +def append_char(var lines : array; c : int) { + return if (empty(lines)) + let li = length(lines) - 1 + return if (length(lines[li]) >= WRAP_CHARS) + lines[li] = "{lines[li]}{to_char(c)}" +} + +//! a new line after the last, unless the box is full +def new_line(var lines : array) { + return if (length(lines) >= MAX_LINES) + lines |> push("") +} + +//! the last character off the last line; an empty last line goes away unless it is the only one +def erase_last(var lines : array) { + return if (empty(lines)) + let li = length(lines) - 1 + let n = length(lines[li]) + if (n == 0) { + if (li > 0) { + lines |> pop() + } + return + } + lines[li] = n == 1 ? "" : clone_string(slice(lines[li], 0, n - 1)) // never a zero-length view of the string being replaced +} + +//! the box's lines from a text, one per line break, never empty +def lines_of(text : string) : array { + var lines <- split(text, "\n") + if (empty(lines)) { + lines |> push("") + } + return <- lines +} + +//! inside a label's box (left, top, right, bottom relative to the pen at x, y), with a margin around it +def inside_box(box : float4; x, y, px, py, pad : float) : bool { + return px >= x + box.x - pad && px <= x + box.z + pad && py >= y + box.y - pad && py <= y + box.w + pad +} diff --git a/examples/dasLLAMA/parrot/web_shell.html b/examples/dasLLAMA/parrot/web_shell.html new file mode 100644 index 0000000000..81af8724bf --- /dev/null +++ b/examples/dasLLAMA/parrot/web_shell.html @@ -0,0 +1,300 @@ + + + + + +parrot — dasllama.io + + + + + + + + + + + + + +
+ + +
+

parrot

+

reading the model list...

+
+ +

The page asks for the microphone when you press record. Your recording stays in this tab: nothing is uploaded, and the voice is gone when you close it.

+ +
+
+ + + + + diff --git a/examples/dasLLAMA/storyteller/web_shell.html b/examples/dasLLAMA/storyteller/web_shell.html index a8ff586c5d..0b451789da 100644 --- a/examples/dasLLAMA/storyteller/web_shell.html +++ b/examples/dasLLAMA/storyteller/web_shell.html @@ -115,6 +115,11 @@

storyteller

return span.innerHTML; } + // a page the browser brings back from its back-forward cache returns with the program's + // workers and the audio output frozen mid-frame and out of step with each other, and the + // first thing heard is whatever sat in the output ring: such a page starts over instead + window.addEventListener('pageshow', function (e) { if (e.persisted) location.reload(); }); + if (!SUPPORTED) { if (FORCE === 'unsupported' || !HAS_WASM64) { showNote('needs a memory64 browser', diff --git a/examples/dasLLAMA/storywish/.das_package b/examples/dasLLAMA/storywish/.das_package index 75b726bcd3..08fd8fc56c 100644 --- a/examples/dasLLAMA/storywish/.das_package +++ b/examples/dasLLAMA/storywish/.das_package @@ -5,7 +5,7 @@ require daslib/daspkg [export] def package() { package_name("storywish") - package_description("Storywish: type the words, a story model trained to take requests writes a tale that uses them while KittenTTS reads it aloud - dasGlfw + dasOpenGL + dasAudio over dasLLAMA") + package_description("Storywish: type the words, a story model trained to take requests writes a tale that uses them while Pocket TTS reads it aloud - dasGlfw + dasOpenGL + dasAudio over dasLLAMA") } [export] diff --git a/examples/dasLLAMA/storywish/main.das b/examples/dasLLAMA/storywish/main.das index 35a415b34f..3e968d7794 100644 --- a/examples/dasLLAMA/storywish/main.das +++ b/examples/dasLLAMA/storywish/main.das @@ -23,16 +23,17 @@ require wish // the request side: typed line -> words -> prompt, // Storywish. You type the words you wish for - "dragon, cake, moon" - and Enter asks a story // model trained to take requests (tinystories-instruct-27M, a llama trained on the // TinyStoriesInstruct corpus) for a children's story that uses them; it writes on screen a few -// tokens per frame while KittenTTS reads each finished sentence aloud. Tab asks for dialogue in +// tokens per frame while Pocket TTS reads each finished sentence aloud. Tab asks for dialogue in // the story. Enter while a story is being told stops it and tells the next one; the sentences of // the old story still queued for speech are skipped. Escape quits. // // bin/daslang -jit examples/dasLLAMA/storywish/main.das -- --models [--words "dragon, cake, moon"] // -// holds the two models and the front-end packs - tts_g2p.bin (or its American-only twin -// tts_g2p_en_us.bin, which the web set ships) and tts_postag.bin. A model is its gguf (the -// default names) or a prepared .dlim image baked for the running build's identity -// (dasllama-convert --config) - the web build ships images only, the shell names them. +// holds the two models. The story model is its gguf (the default name) or a prepared +// .dlim image baked for the running build's identity (dasllama-convert --config) - the web +// build ships the image, the shell names it. The speech model is a Pocket TTS file with one +// voice and no codec encoder (the shipped form: it reads text, needs no packs, cannot clone); +// any TTS file the facade loads works here, a Kitten or Kokoro one with its packs beside it. // The language model runs on the frame thread in per-frame token budgets, so the loop never // blocks; speech synthesis runs on its own thread, fed sentences through one stream and // answering with PCM through another, and the frame thread plays the clips back to back. @@ -40,14 +41,14 @@ require wish // the request side: typed line -> words -> prompt, [CommandLineArgs] struct WishArgs { @clarg_short = "m" - @clarg_doc = "Directory holding the story model, the TTS model and its phoneme packs (default: the current directory)" + @clarg_doc = "Directory holding the story model and the TTS model (default: the current directory)" models : string = "." @clarg_doc = "The story model file inside --models" story_model : string = "tinystories-instruct-27M-Q8_0.gguf" @clarg_doc = "The TTS model file inside --models" - tts_model : string = "kitten-nano.gguf" + tts_model : string = "pocket-tts-en-stuart-kq.gguf" @clarg_doc = "Voice name or alias (default: the model's last voice)" voice : string diff --git a/examples/dasLLAMA/storywish/models.json b/examples/dasLLAMA/storywish/models.json index 48dc96a770..0a573a2a25 100644 --- a/examples/dasLLAMA/storywish/models.json +++ b/examples/dasLLAMA/storywish/models.json @@ -1,10 +1,8 @@ { "images": [ - { "file": "tinystories-instruct-27M-Q8_0.gguf", "repo": "borisbat/dasllama-stories", "sha256": "92c2b775070b76ee31b9921f7f339ec9fc0e08b54be0616418414d4eec65d282", "dlim": "tinystories-instruct-27M.dlim" }, - { "file": "kitten-nano.gguf", "repo": "borisbat/dasllama-tts", "sha256": "4556948c36a29e4be5ad521e597a20ae817059c404fea3c5d935afa73506d9da", "dlim": "kitten-nano.dlim" } + { "file": "tinystories-instruct-27M-Q8_0.gguf", "repo": "borisbat/dasllama-stories", "sha256": "92c2b775070b76ee31b9921f7f339ec9fc0e08b54be0616418414d4eec65d282", "dlim": "tinystories-instruct-27M.dlim" } ], - "packs": [ - { "file": "tts_g2p_en_us.bin", "repo": "borisbat/dasllama-tts", "sha256": "6f69d2e74565bd7d876b8d1f4042bf8c1c5b615387fa26ff45215cf447932154" }, - { "file": "tts_postag.bin", "repo": "borisbat/dasllama-tts", "sha256": "38c2e85f7fef3e57d561d2aa0af25fccda4276376ba1993c3dbc2ae0ebfa57b4" } + "files": [ + { "file": "pocket-tts-en-stuart-kq.gguf", "repo": "borisbat/dasllama-tts", "sha256": "bc9604b527066134354dc480e20c960f63f5c3538c1dd757ba409bd782cddac9" } ] } diff --git a/examples/dasLLAMA/storywish/web_shell.html b/examples/dasLLAMA/storywish/web_shell.html index ef6f8f4d56..5f18c03066 100644 --- a/examples/dasLLAMA/storywish/web_shell.html +++ b/examples/dasLLAMA/storywish/web_shell.html @@ -51,7 +51,7 @@ byte. Then it reads models/manifest.json - the list the deploy minted, stamped with the IMAGE_VERSION it minted at - and refuses a set minted for another version before fetching a megabyte of it: the program would decline the images anyway, this way the reason is on the - page. Otherwise it fetches the story model, the TTS model and its phoneme packs into MEMFS, + page. Otherwise it fetches the story model and the speech model into MEMFS, then waits for a click - browsers only let audio start from a gesture. Models come from ./models/ beside the page, or ?models=. ?force=unsupported shows the browser note on a supporting browser, for checking the page. A program that aborts after the click puts its @@ -114,6 +114,11 @@

storywish

return span.innerHTML; } + // a page the browser brings back from its back-forward cache returns with the program's + // workers and the audio output frozen mid-frame and out of step with each other, and the + // first thing heard is whatever sat in the output ring: such a page starts over instead + window.addEventListener('pageshow', function (e) { if (e.persisted) location.reload(); }); + if (!SUPPORTED) { if (FORCE === 'unsupported' || !HAS_WASM64) { showNote('needs a memory64 browser', @@ -210,7 +215,7 @@

storywish

}; return c; })(), - arguments: ['--models', '/models', '--story-model', 'tinystories-instruct-27M.dlim', '--tts-model', 'kitten-nano.dlim', '--autoplay'], + arguments: ['--models', '/models', '--story-model', 'tinystories-instruct-27M.dlim', '--tts-model', 'pocket-tts-en-stuart-kq.gguf', '--autoplay'], print: function (text) { console.log(text); remember(text); }, printErr: function (text) { console.error(text); remember(text); }, onAbort: function (what) { showFailure(what || 'abort'); }, diff --git a/examples/dasLLAMA/wasm/mint_models.py b/examples/dasLLAMA/wasm/mint_models.py index 33bbd694ae..70768748b3 100644 --- a/examples/dasLLAMA/wasm/mint_models.py +++ b/examples/dasLLAMA/wasm/mint_models.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 -"""Stage a browser example's model set: fetch the GGUFs and packs its models.json names from -Hugging Face, mint each GGUF into a .dlim against the wasm64 build's DlimConfiguration, copy the -packs, and write models/manifest.json - the list the example's web shell reads, stamped with the -IMAGE_VERSION the images carry. +"""Stage a browser example's model set: fetch the files its models.json names from Hugging Face, +mint each GGUF under `images` into a .dlim against the wasm64 build's DlimConfiguration, copy the +`packs` and `files` as they are (a front-end pack; a Pocket TTS GGUF, which is its own served +form), copy each `tree` file from the repository itself (a checked-in model such as the +voice-activity weights, named by its repo-relative path), and write models/manifest.json - the +list the example's web shell reads, stamped with the IMAGE_VERSION the images carry (a set with no +image carries the version the deploy expects). mint_models.py --example examples/dasLLAMA/storywish --config wasm64.json \ --daslang bin/daslang --out _site/examples/storywish/models \ @@ -74,6 +77,11 @@ def main(): spec = json.load(f) os.makedirs(a.out, exist_ok=True) files, versions = [], set() + for key in ("images", "packs", "files", "tree"): + if key in spec and not spec[key]: + raise SystemExit(f"models.json names an empty `{key}` list - drop the key or fill it; a set that mints nothing must say so by omission") + if not any(spec.get(key) for key in ("images", "packs", "files", "tree")): + raise SystemExit("models.json stages nothing") for entry in spec.get("images", []): gguf = fetch(entry, a.cache) @@ -92,15 +100,26 @@ def main(): files.append({"name": entry["dlim"], "bytes": os.path.getsize(dlim), "sha256": sha256_of(dlim), "source": f"{entry['repo']}/{entry['file']}"}) print(f"minted {entry['dlim']} ({os.path.getsize(dlim) >> 20} MB, IMAGE_VERSION {version}) from {entry['file']}") - for entry in spec.get("packs", []): + # packs and files ship as they are: a front-end pack, or a GGUF that is its own served form (a Pocket file) + for entry in spec.get("packs", []) + spec.get("files", []): src = fetch(entry, a.cache) dst = os.path.join(a.out, entry["file"]) shutil.copyfile(src, dst) files.append({"name": entry["file"], "bytes": os.path.getsize(dst), "sha256": entry["sha256"], "source": f"{entry['repo']}/{entry['file']}"}) - if len(versions) != 1: + # a file the tree itself carries: copied from the checkout, its hash held like a fetched file's + for entry in spec.get("tree", []): + src = os.path.join(repo_root, entry["path"]) + got = sha256_of(src) + if got != entry["sha256"]: + raise SystemExit(f"{entry['file']}: sha256 {got}, models.json says {entry['sha256']} - the tree file changed; update models.json") + dst = os.path.join(a.out, entry["file"]) + shutil.copyfile(src, dst) + files.append({"name": entry["file"], "bytes": os.path.getsize(dst), "sha256": entry["sha256"], "source": entry["path"]}) + + if len(versions) > 1: raise SystemExit(f"the minted images disagree on IMAGE_VERSION: {sorted(versions)}") - version = versions.pop() + version = versions.pop() if versions else a.expect_image_version if a.expect_image_version and version != a.expect_image_version: sys.stderr.write(f"minted images carry IMAGE_VERSION {version}, the tree says {a.expect_image_version} - the converter that minted them is not this tree's\n") sys.exit(3) # the deploy tells this exit apart: a version mismatch reds the run, every other failure stages a placeholder diff --git a/examples/dasLLAMA/wasm/test_mint_models.py b/examples/dasLLAMA/wasm/test_mint_models.py index 30954b1fc2..c8ce1e90a7 100644 --- a/examples/dasLLAMA/wasm/test_mint_models.py +++ b/examples/dasLLAMA/wasm/test_mint_models.py @@ -169,5 +169,90 @@ def test_a_converter_that_fails_is_reported_with_its_exit(self): self.assertIn("boom", err) +class ServedAsIsTest(unittest.TestCase): + """the lists beside `images`: `files` (a GGUF that is its own served form) and `tree` (a file + the repository carries, named by its repo-relative path) - copied as they are, hash-held""" + + def with_repo(self, fx, spec, tree_bytes=None): + """the spec written, and a stand-in repository root four levels above a fake script path, + the way the script finds the checkout; `tree_bytes` lands at models/vad.bin under it""" + fx.spec = spec + (fx.example / "models.json").write_text(json.dumps(spec)) + repo = fx.root / "repo" + (repo / "models").mkdir(parents=True) + if tree_bytes is not None: + (repo / "models" / "vad.bin").write_bytes(tree_bytes) + return str(repo / "examples" / "dasLLAMA" / "wasm" / "mint_models.py") + + def run_with_file(self, fake_file, argv): + saved = mint_models.__file__ + mint_models.__file__ = fake_file + try: + return run_main(argv) + finally: + mint_models.__file__ = saved + + def test_a_files_entry_is_copied_as_it_is_beside_the_image(self): + with tempfile.TemporaryDirectory() as tmp: + fx = Fixture(tmp, image_version=35) + served = b"a Pocket file, served as it is" + (fx.cache / f"{sha(served)}-pocket.gguf").write_bytes(served) + spec = dict(fx.spec, files=[{"file": "pocket.gguf", "repo": "someone/tts", "sha256": sha(served)}]) + fake_file = self.with_repo(fx, spec) + code, out, err = self.run_with_file(fake_file, fx.argv("--expect-image-version", "35")) + self.assertIsNone(code, err) + manifest = json.loads((fx.out / "manifest.json").read_text()) + self.assertEqual([f["name"] for f in manifest["files"]], ["story.dlim", "pack.bin", "pocket.gguf"]) + self.assertEqual((fx.out / "pocket.gguf").read_bytes(), served) + self.assertEqual(manifest["files"][2]["source"], "someone/tts/pocket.gguf") + + def test_a_tree_entry_is_copied_from_the_checkout_and_named_by_its_path(self): + with tempfile.TemporaryDirectory() as tmp: + fx = Fixture(tmp, image_version=35) + vad = b"voice activity weights" + spec = dict(fx.spec, tree=[{"file": "vad.bin", "path": "models/vad.bin", "sha256": sha(vad)}]) + fake_file = self.with_repo(fx, spec, vad) + code, out, err = self.run_with_file(fake_file, fx.argv("--expect-image-version", "35")) + self.assertIsNone(code, err) + manifest = json.loads((fx.out / "manifest.json").read_text()) + self.assertEqual((fx.out / "vad.bin").read_bytes(), vad) + self.assertEqual(manifest["files"][-1], {"name": "vad.bin", "bytes": len(vad), "sha256": sha(vad), "source": "models/vad.bin"}) + + def test_a_tree_file_whose_hash_moved_stops_the_run(self): + with tempfile.TemporaryDirectory() as tmp: + fx = Fixture(tmp, image_version=35) + spec = dict(fx.spec, tree=[{"file": "vad.bin", "path": "models/vad.bin", "sha256": sha(b"the weights the page was built for")}]) + fake_file = self.with_repo(fx, spec, b"other weights checked in since") + code, out, err = self.run_with_file(fake_file, fx.argv()) + self.assertIsInstance(code, str) + self.assertIn("vad.bin: sha256", code) + self.assertIn("the tree file changed", code) + self.assertFalse((fx.out / "manifest.json").exists()) + + def test_a_set_with_no_image_carries_the_version_the_deploy_expects(self): + with tempfile.TemporaryDirectory() as tmp: + fx = Fixture(tmp) + served = b"a Pocket file" + (fx.cache / f"{sha(served)}-pocket.gguf").write_bytes(served) + spec = {"files": [{"file": "pocket.gguf", "repo": "someone/tts", "sha256": sha(served)}]} + fake_file = self.with_repo(fx, spec) + code, out, err = self.run_with_file(fake_file, fx.argv("--expect-image-version", "41", "--stamp-page", str(fx.page))) + self.assertIsNone(code, err) + self.assertEqual(json.loads((fx.out / "manifest.json").read_text())["image_version"], 41) + self.assertIn("/* @image-version */ 41", fx.page.read_text()) + self.assertFalse((fx.out / "story.dlim").exists(), "nothing was minted") + + def test_a_set_with_no_image_and_no_expected_version_is_unstamped(self): + with tempfile.TemporaryDirectory() as tmp: + fx = Fixture(tmp) + served = b"a Pocket file" + (fx.cache / f"{sha(served)}-pocket.gguf").write_bytes(served) + fake_file = self.with_repo(fx, {"files": [{"file": "pocket.gguf", "repo": "someone/tts", "sha256": sha(served)}]}) + code, out, err = self.run_with_file(fake_file, fx.argv("--stamp-page", str(fx.page))) + self.assertIsNone(code, err) + self.assertEqual(json.loads((fx.out / "manifest.json").read_text())["image_version"], 0) + self.assertIn("/* @image-version */ 0", fx.page.read_text(), "0 is the unstamped slot the page reads as no check") + + if __name__ == "__main__": unittest.main() diff --git a/modules/dasAudio/CMakeLists.txt b/modules/dasAudio/CMakeLists.txt index 587110c9b4..fb41865186 100644 --- a/modules/dasAudio/CMakeLists.txt +++ b/modules/dasAudio/CMakeLists.txt @@ -20,6 +20,8 @@ IF ((NOT DAS_AUDIO_INCLUDED) AND ((NOT ${DAS_AUDIO_DISABLED}) OR (NOT DEFINED DA execute_process(COMMAND ${CMAKE_COMMAND} -DMINIAUDIO_H=${miniaudio_SOURCE_DIR}/miniaudio.h -P ${DAS_AUDIO_DIR}/patches/miniaudio_memory64.cmake) + # a change to the patch script reconfigures, so an already-fetched header receives it + SET_PROPERTY(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${DAS_AUDIO_DIR}/patches/miniaudio_memory64.cmake) SET(AUDIO_INCLUDE_DIR ${DAS_AUDIO_DIR}/src ${miniaudio_SOURCE_DIR}) SET(CIPIC_HRTF_INCLUDE_DIR ${DAS_AUDIO_DIR}/cipic-hrtf/include) diff --git a/modules/dasAudio/patches/miniaudio_memory64.cmake b/modules/dasAudio/patches/miniaudio_memory64.cmake index c3e829136c..32a8687b27 100644 --- a/modules/dasAudio/patches/miniaudio_memory64.cmake +++ b/modules/dasAudio/patches/miniaudio_memory64.cmake @@ -20,13 +20,13 @@ endif() file(READ "${MINIAUDIO_H}" _ma) -# Idempotency guard: key on the LAST-added patch marker (the worklet non-blocking -# patch), not the first (toPtr). Otherwise a tree already patched by an older -# version of this script (toPtr only) would early-return and never receive a -# newly-added block. With this marker, a toPtr-only tree still runs the script; -# the toPtr string(REPLACE)s are no-ops (targets already gone) and only the new -# worklet block applies. -string(FIND "${_ma}" "daslang non-blocking patch (see miniaudio_memory64.cmake)" _already) +# Idempotency guard: key on the LAST-added patch marker (the capture-rate patch), +# not the first (toPtr). Otherwise a tree already patched by an older version of +# this script would early-return and never receive a newly-added block. With this +# marker, an older-patched tree still runs the script; the earlier string(REPLACE)s +# are no-ops (targets already gone) and only the new block applies (block 6b is +# the form of block 6 that targets a tree carrying the previous non-blocking text). +string(FIND "${_ma}" "daslang capture-rate patch" _already) if(_already GREATER -1) message(STATUS "miniaudio_memory64.cmake: already patched, skipping ${MINIAUDIO_H}") return() @@ -78,6 +78,14 @@ string(REPLACE # Only compiled when MA_USE_AUDIO_WORKLETS (the threaded web build); inert # otherwise. Verified end-to-end: a 440Hz worklet tone plays in Chrome on # memory64+pthread+wasm-EH with no asyncify. +# +# The descriptors' sample rate is the context's own (the capture-rate patch): the +# generic ma_device_init reads the native rate back from the descriptor and sets up +# resampling between it and the requested one, and a capture context is created at +# the browser's rate (48 kHz in Chrome), not the requested one - left at the +# requested value, a 24 kHz capture arrives as 48 kHz frames counted as 24 kHz, +# an octave down and twice as long. A playback context is created at the +# requested rate, so its descriptor reads back the same value as before. string(REPLACE [==[ while (pDevice->webaudio.initResult == MA_BUSY) { emscripten_sleep(1); } /* We must wait for initialization to complete. We're just spinning here. The emscripten_sleep() call is why we need to build with `-sASYNCIFY`. */ @@ -89,12 +97,16 @@ string(REPLACE }]==] [==[ /* daslang non-blocking patch (see miniaudio_memory64.cmake): drop the emscripten_sleep busy-wait that forces -sASYNCIFY. Pre-fill descriptors - from config; the worklet connects asynchronously. */ + from config; the worklet connects asynchronously. The rate is the + context's own (daslang capture-rate patch): a capture context is created + at the browser's rate, and the device layer resamples to the requested one. */ { + ma_uint32 awRate = (ma_uint32)EM_ASM_INT({ return emscriptenGetAudioObject($0).sampleRate; }, pDevice->webaudio.audioContext); ma_uint32 awCh = (pDescriptorPlayback != NULL && pDescriptorPlayback->channels > 0) ? pDescriptorPlayback->channels : MA_DEFAULT_CHANNELS; if (pDescriptorPlayback != NULL) { pDescriptorPlayback->format = ma_format_f32; pDescriptorPlayback->channels = awCh; + if (awRate != 0) { pDescriptorPlayback->sampleRate = awRate; } ma_channel_map_init_standard(ma_standard_channel_map_webaudio, pDescriptorPlayback->channelMap, ma_countof(pDescriptorPlayback->channelMap), pDescriptorPlayback->channels); pDescriptorPlayback->periodSizeInFrames = 128; pDescriptorPlayback->periodCount = 1; @@ -103,6 +115,7 @@ string(REPLACE ma_uint32 awCapCh = (pDescriptorCapture->channels > 0) ? pDescriptorCapture->channels : MA_DEFAULT_CHANNELS; pDescriptorCapture->format = ma_format_f32; pDescriptorCapture->channels = awCapCh; + if (awRate != 0) { pDescriptorCapture->sampleRate = awRate; } ma_channel_map_init_standard(ma_standard_channel_map_webaudio, pDescriptorCapture->channelMap, ma_countof(pDescriptorCapture->channelMap), pDescriptorCapture->channels); pDescriptorCapture->periodSizeInFrames = 128; pDescriptorCapture->periodCount = 1; @@ -117,6 +130,35 @@ string(REPLACE }]==] _ma "${_ma}") +# 6b) The capture-rate patch on a tree the previous version of this script already +# patched (its block 6 text is present, without the rate): the same result as block 6. +# On a fresh tree block 6 has already written the rate lines, so none of these match. +string(REPLACE +[==[ from config; the worklet connects asynchronously. */ + { + ma_uint32 awCh = (pDescriptorPlayback != NULL && pDescriptorPlayback->channels > 0) ? pDescriptorPlayback->channels : MA_DEFAULT_CHANNELS;]==] +[==[ from config; the worklet connects asynchronously. The rate is the + context's own (daslang capture-rate patch): a capture context is created + at the browser's rate, and the device layer resamples to the requested one. */ + { + ma_uint32 awRate = (ma_uint32)EM_ASM_INT({ return emscriptenGetAudioObject($0).sampleRate; }, pDevice->webaudio.audioContext); + ma_uint32 awCh = (pDescriptorPlayback != NULL && pDescriptorPlayback->channels > 0) ? pDescriptorPlayback->channels : MA_DEFAULT_CHANNELS;]==] + _ma "${_ma}") +string(REPLACE +[==[ pDescriptorPlayback->channels = awCh; + ma_channel_map_init_standard(]==] +[==[ pDescriptorPlayback->channels = awCh; + if (awRate != 0) { pDescriptorPlayback->sampleRate = awRate; } + ma_channel_map_init_standard(]==] + _ma "${_ma}") +string(REPLACE +[==[ pDescriptorCapture->channels = awCapCh; + ma_channel_map_init_standard(]==] +[==[ pDescriptorCapture->channels = awCapCh; + if (awRate != 0) { pDescriptorCapture->sampleRate = awRate; } + ma_channel_map_init_standard(]==] + _ma "${_ma}") + # 7) AudioWorklet dangling-config fix. The processor-created callback runs long # after ma_device_init returns (non-blocking, block 6), so pConfig — a local of # the generic ma_device_init — is dead. Read the device type from the long-lived diff --git a/modules/dasLLAMA/ARCHITECTURE_POCKET.md b/modules/dasLLAMA/ARCHITECTURE_POCKET.md index 641fb2983c..cf3cc9890d 100644 --- a/modules/dasLLAMA/ARCHITECTURE_POCKET.md +++ b/modules/dasLLAMA/ARCHITECTURE_POCKET.md @@ -14,9 +14,11 @@ The TTS block home, facade and phoneme families are `ARCHITECTURE_TTS.md`. assembly. The weight map of the converted GGUF (`harness/convert_pocket.py`: the canonical tensor names `backbone.N.*`, `head.*`, `mimi.enc_tf.N.*` / `mimi.dec_tf.N.*`, the rest as the bundle names them; the `pocket.*` scalars from the package's per-language config; the - unigram tokenizer under `tokenizer.ggml.model = "t5"`; the roster's clips as `voice.` - PCM tensors), the model (`PocketModel`: the causal backbone, the one-step flow head, the - Mimi-derived codec, the roster and its encoded voice states), the activation carrier + unigram tokenizer under `tokenizer.ggml.model = "t5"`; the roster as `voice_latents.` + latent frames - an older file carries `voice.` PCM instead, encoded on first use; + `pocket.cloning` says whether the codec encoder is inside), the model (`PocketModel`: the + causal backbone, the one-step flow head, the Mimi-derived codec with or without its encoder, + the roster and its voice states built from the stored frames), the activation carrier (`PocketScratch`), and the assembly - the voice prompt (sec.2.47), the text prompt, the frame loop (sec.2.48), the codec decoder over a chunk's latents (sec.2.46) - plus the reference driver's text preparation and chunker (sec.2.49). `pocket_speak` is the facade's entry; the @@ -55,8 +57,13 @@ The caches are sized for the clip plus 1024 rows and grow, the voice's rows kept text plus every frame its cap allows needs more - one unsplittable run of two hundred tokens is such a chunk. A clip is at most 60 s (`POCKET_MAX_VOICE_SECONDS`): the state is the clip's frames per layer, and the codec encoder's attention is a query block by the 250-key window it sees. -The roster's clips ride the GGUF and encode on first use; a cloned voice is the same path over a -caller's clip (`tts_register_voice`). The package's precomputed states differ from the clip path +The roster rides the GGUF as each clip's latent frames (`voice_latents.`, the package's +own codec encoder over the clip at conversion), and a voice's state is built from them on first +use - the second half of the clip path, no encoder needed; a file of the older form carries the +clips themselves (`voice.`) and encodes them on first use. A cloned voice is the whole clip +path over a caller's clip (`tts_register_voice`), so it needs the encoder: a file converted +`--no-cloning` leaves the encoder out, says so in `pocket.cloning`, reports `cloning = false` in +`caps()` and refuses a clip by name. The package's precomputed states differ from the clip path by 1.5e-2 (they come from another checkpoint revision; `harness/pocket_oracle.py` dumps both and `test_pocket_parity`'s voice cell compares the clip path); the clip path is the reference. @@ -91,8 +98,21 @@ another language takes the text as it is, since the normalizer reads English. ### 2.50 The published file carries the served quants {#pocket-q8-file} -Two lanes, as the StyleTTS2 families have: f32, the parity rail's reference, and q8, the -served default - the transformer layers' four matrices, the frame input projection and every +A file has three lanes and its formats decide which it can take. A K-quant dense layer +(`convert_pocket.py --kq`: the backbone's and the codec transformers' matrices as Q4_K, the flow +head as Q8_0, the rest as the q8 form writes it) serves as its own planes unless a lane is +pinned - `TtsLinear` holds the plane pair the GGUF transcoder wrote, repacked where the backend +the load selected carries kq kernels (the load line says which arm), and the frame loop's GEMV +and the prompt's GEMM take the engine's own K-quant entries (`linear_rows_decode`, +`linear_rows_kq`), the rows requantized to the Q8_K form the way the engine's own decode does. +The text embedding table is Q4_K on disk only: it is a lookup, and dequantizes at load on every +lane. Pinned q8 or f32, a K-quant tensor dequantizes into that lane, so one file serves every +lane and the rig compares them on the same sentences. A vector layer the file stores Q8_0 (the +head) runs its GEMV on the q8 lane. + +The two lanes every Pocket file has, as the StyleTTS2 families have them: f32, the parity +rail's reference, and q8, the served default - the transformer layers' four matrices, the frame +input projection and every dense stride-1 codec conv on 32-wide channels as Q8_0 rows (`linear_prepare`, `conv1d_q8_eligible`), the decode step on the q8 GEMV entry. The published GGUF (`convert_pocket.py --q8`) stores exactly those tensors as Q8_0 in the layout the kernels read diff --git a/modules/dasLLAMA/ARCHITECTURE_TTS.md b/modules/dasLLAMA/ARCHITECTURE_TTS.md index 679fea547c..139c67ebe0 100644 --- a/modules/dasLLAMA/ARCHITECTURE_TTS.md +++ b/modules/dasLLAMA/ARCHITECTURE_TTS.md @@ -43,14 +43,13 @@ TTS files implement (sec.2.28-2.35, 2.43). `ARCHITECTURE_COMMON.md` (repo root) and takes no rewrite, which matters because the rewrite is not the identity on one: it reads the DRESS vowel before a linking rhotic as SQUARE, having nothing in the string to tell merry from Mary. A vowel the two lexicons give no evidence for before a dropped rhotic keeps that - rhotic rather than losing it. The bath-trap split reaches only lexicon - words. Loads a phoneme pack - `tts_g2p.bin` (both dialect tiers) or `tts_g2p_en_us.bin` (the - American tier alone, sec.2.43) - pack - version 2 (`harness/build_g2p_data.py`: the gold tier extended by + rhotic rather than losing it. The bath-trap split reaches only lexicon words. Loads a phoneme + pack - `tts_g2p.bin` (both dialect tiers) or `tts_g2p_en_us.bin` (the American tier alone, + sec.2.43) - pack version 2 (`harness/build_g2p_data.py`: the gold tier extended by `harness/g2p_local_additions.json`, the US and GB keys merged into one string table per tier, the GRU stored as f16, CMUdict pruned of the words both dialects' lexicons carry - - safe because the fallback reads the lexicon first), - searched in place as byte-sorted string tables; a version 1 pack is refused by name. The + safe because the fallback reads the lexicon first), searched in place as byte-sorted string + tables; a version 1 pack is refused by name. The 200-sentence fixtures under `tests/_tts_fixtures/` (American, minted by `harness/mint_tts_g2p_fixture.py` from the G2P fidelity experiment; British, minted by `harness/mint_tts_g2p_gb_fixture.py` from the reference's own British front end) are the @@ -74,7 +73,8 @@ TTS files implement (sec.2.28-2.35, 2.43). `ARCHITECTURE_COMMON.md` (repo root) squared; a cache grows with its fill kept), rope over rows, per-channel layer scale and the replicate left pad. A weight is an ONNX-layout array plus the served layout `conv1d_prepare` / `linear_prepare` mint for the consumer the reader names (`served_rows`, - `rows_only`, `vec_only`), the unread one dropped; beside every weight array sits its `TtsSpan` into the + `rows_only`, `vec_only`), the unread one dropped - or the file's own K-quant planes + (`linear_take_kq`), the kq lane beside f32 and q8; beside every weight array sits its `TtsSpan` into the model's blob, and `weights_walk` is the one walk that moves weights into a staging blob or binds them as borrowed views over a served plane (`release_weight` is the one teardown). One home: the block home holds the operators, and it names no family type. diff --git a/modules/dasLLAMA/ENVIRONMENT.md b/modules/dasLLAMA/ENVIRONMENT.md index f953b008c9..05412c13e7 100644 --- a/modules/dasLLAMA/ENVIRONMENT.md +++ b/modules/dasLLAMA/ENVIRONMENT.md @@ -247,7 +247,7 @@ Apple Accelerate / AMX float lane. `DASLLAMA_ACCEL` arms the whole group. | `DASLLAMA_TEST_FAMILY` | text | unset | Comma-separated filter restricting which model families run. | | `DASLLAMA_LLAMA2C_DIR` | path | unset | Directory of llama2.c reference checkpoints for the forward/decode parity tests. | | `DASLLAMA_WHISPER_DIR` | path | unset | Directory of whisper models for the audio tests. | -| `DISPLAY` | text | unset | Ambient platform variable; read only by the browser examples' smoke tests (storyteller, storywish) to tell whether a Linux box has a window server for the example's window. | +| `DISPLAY` | text | unset | Ambient platform variable; read only by the browser examples' smoke tests (storyteller, storywish, parrot) to tell whether a Linux box has a window server for the example's window. | | `WAYLAND_DISPLAY` | text | unset | Ambient platform variable (the Wayland twin of DISPLAY); read only by the browser examples' smoke tests. | | `DASLLAMA_CORPUS_DIR` | path | unset | Directory of audio corpus files for the transcription tests. | | `TMPDIR` | path | /tmp | Scratch directory for test artifacts; set by the OS on macOS. | diff --git a/modules/dasLLAMA/PERF_LEDGER.md b/modules/dasLLAMA/PERF_LEDGER.md index 01c6973e34..d5f8fa930f 100644 --- a/modules/dasLLAMA/PERF_LEDGER.md +++ b/modules/dasLLAMA/PERF_LEDGER.md @@ -1351,3 +1351,24 @@ commits: direction-grade. grows `mtp_cat` to 2 x 9 x dim floats (Qwen3.8-27B: 370 KB) and `mtp_logits_b` to 9 x vocab floats once (about 5 MB). Decision: taken - the round's gain rides on the drafter's presence, and the sidecar is a fraction of a percent of the target it drafts for. + +### From the Pocket small form (2026-09-10) + +Instruments: `harness/tts_synth.das` (`-jit -module-cache`, alba, the first 60 sentences of the +rig corpus, M1 Max, the box's tune profile, JIT cache warm) with the process's resident set read +through `ps -o rss` every half second, two reps per file; `harness/tts_rig.py` (alba, the 200 +sentences, parakeet WER + UTMOS) per weight lane. Every pair here is two processes on one box: +direction-grade. + +- **The English file, the q8 form against the small form (`pocket-tts-en-q8.gguf` 152 MB, + `pocket-tts-en-kq.gguf` 75 MB; the one-voice `pocket-tts-en-stuart-kq.gguf` 65 MB):** the + compiled program before the load reads 1.87 GB on both; the load adds 0.66 GB on the q8 file + and 0.50 on the kq file; over the synthesis the q8 process holds 2.54 GB and the kq process + 2.70 - the kq lane steps up 0.33 GB at its first synthesis and stays there, the q8 lane does not + (reps within 2 MB). RTF over the 60 sentences: q8 0.0510 / 0.0507, kq 0.0435 / 0.0434 - the + small file decodes 15% faster and its process is 6% larger. The step is followup 129's. +- **The kq file's three lanes on the rig (the q8 file 3.91 / 4.328, its f32 lane through the f16 + file 4.32 / 4.366):** native 3.86 / 4.295, the q8 pin 3.73 / 4.295, the f32 pin 3.86 / 4.330. + The lanes agree within the rig's own spread; the small form loses nothing the rig can hear. + Decision: taken - the browser pages read the small forms (storywish the one-voice file, parrot + the 19-voice one), the q8 file stays the desktop default of the served set. diff --git a/modules/dasLLAMA/REVIEW.das b/modules/dasLLAMA/REVIEW.das index 5ab009a70e..f6cde0d661 100644 --- a/modules/dasLLAMA/REVIEW.das +++ b/modules/dasLLAMA/REVIEW.das @@ -545,7 +545,7 @@ let private IMAGE_FILE = "modules/dasLLAMA/dasllama/dasllama_image.das" // in file order. A closure change with IMAGE_VERSION unmoved is red; the finding prints the // value to re-stamp with. let private IMAGE_LAYOUT_STAMP_VERSION = 36 -let private IMAGE_LAYOUT_STAMP_HASH = 0x7665c61f19ad379ful +let private IMAGE_LAYOUT_STAMP_HASH = 0xbadc54ac04cbcad1ul // The helpers that decide WHERE bytes land: the page pad, the plane and total sizing, the // writer's append / zero-fill / header patch, and the header's scalar stores. Changing one diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index 1bbd82f5ff..8be9cb15e8 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -1,10 +1,11 @@ # dasLLAMA Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -docs: `ARCHITECTURE.md`, `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_RUNTIME.md`, `ARCHITECTURE_MEASUREMENT.md` -(the other companions belong to the routed checklists). Planned work: `followup_general.md`, -`followup_vulkan.md`, `followup_metal.md` (the Metal tier, and CPU work measured on macOS), -`PERF_LEDGER.md` (performance goes to the perf ledger, everything else to the followup ledgers). +docs: `ARCHITECTURE.md`, `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_RUNTIME.md`, +`ARCHITECTURE_MEASUREMENT.md` (the other companions belong to the routed checklists). Planned +work: `followup_general.md`, `followup_vulkan.md`, `followup_metal.md` (the Metal tier, and CPU +work measured on macOS), `PERF_LEDGER.md` (performance goes to the perf ledger, everything else +to the followup ledgers). **A dasLLAMA `[test]` file, wherever the diff puts it, answers to this module's `tests/REVIEW.md`.** @@ -78,8 +79,9 @@ function does not thereby pick up the other modality's checklist. file - one stage of the pass that turns text into phonemes (`dasllama/dasllama_textnorm.das`, `dasllama/dasllama_postag.das`, `dasllama/dasllama_g2p.das`) - the front-end packs' mint (`harness/build_g2p_data.py`, `harness/train_postag.py`, `harness/mint_postag_silver.py`, -`performance/build_tts_data.das`), or a call that pins the TTS weight lane (`set_tts_q8` / -`set_styletts2_q8`), wherever the diff puts it, applies `REVIEW_TTS.md`.** +`performance/build_tts_data.das`), the Pocket converter and its card (`harness/convert_pocket.py`, +`harness/tts_model_card.md`), or a call that pins a TTS weight lane (`set_tts_q8` / +`set_styletts2_q8` / `set_pocket_q8`), wherever the diff puts it, applies `REVIEW_TTS.md`.** **A diff that adds a file under `dasllama/`, or adds or moves a def, a `require`, or a module global in a file under `dasllama/`, applies `REVIEW_PLACEMENT.md`** - the what-lands-where rules. @@ -230,8 +232,7 @@ check licenses no names, the line says so. **Checked-in text under `modules/dasLLAMA/` - docs, comments, or string data, any language - that describes a mechanism of the reference build, or names that build, its binaries or its symbols, wherever the diff puts it, applies `REVIEW_UPSTREAM.md`.** The reference build is the -third-party engine this module measures itself against - the checkout -`benchmarks/setup_lcpp_ref.das` pins. +third-party engine this module measures itself against - `benchmarks/setup_lcpp_ref.das` pins it. **A diff that changes what authoring a new weight format entails - a step added or dropped, a file the author must touch, a fixture or probe entry the format must supply, or a gate it must @@ -239,10 +240,9 @@ pass - updates `HOW_TO_ADD_A_FORMAT.md` in the same change.** The how-to is the author's whole brief: a step dropped there is a step the next format silently skips. **Legal attribution - a third party's copyright line, licence name, or licence text - lives in -`THIRD_PARTY_NOTICES.md`, in the `LICENSE.*` files, in a model card - the provenance-and-licence -page published beside a released model or pack - or in a ledger row naming a licence as a -reason to adopt or reject a model, a dataset, or a dependency; anywhere else in prose it is a -defect.** +`THIRD_PARTY_NOTICES.md`, the `LICENSE.*` files, a model card (the provenance-and-licence page +published beside a released model or pack), or a ledger row naming a licence as a reason to +adopt or reject a model, a dataset, or a dependency; anywhere else in prose it is a defect.** **A def of a facade file - one whose defs reach a consumer through `require dasllama/dasllama`; `REVIEW.das`'s `FACADE_FILES` is the list - and a new OVERLOAD of one, is TAUGHT: demonstrated @@ -262,12 +262,12 @@ the renderer emits but the registry does not is caught by `tests/test_env_regist **Hand-editing `dasllama/dasllama_unicode.das`'s RANGES/WS tables is a defect - regenerate them by retranscoding `$LCPP/src/unicode-data.cpp` (the reference checkout) instead.** -**A diff that adds a file under `dasllama/`, moves a def, a `require`, or a module global -between files there, or changes what a file owns, lands the sec.1 edit that keeps the -charters true - in an `ARCHITECTURE_*.md` companion, never `ARCHITECTURE.md` - in the same -change.** A diff that adds a file to any folder where another file has its own -sec.1 charter line lands the new file's charter line too. A module-root doc file - a ledger, a -plan - has no charter line and needs no charter edit. +**A diff that adds a file under `dasllama/`, adds or moves a def, a `require`, or a module +global in a file there, or gives a file a weight format, a serving lane (the quant form a +tensor serves from) or a data structure its charter does not name, lands the sec.1 edit that +keeps the charters true - in an `ARCHITECTURE_*.md` companion, never `ARCHITECTURE.md` - in the +same change.** A diff that adds a file to any folder where another file has its own sec.1 +charter line lands the new file's charter line too; a module-root ledger has no charter line. **A diff that adds, removes, or moves a section of an `ARCHITECTURE_*.md` companion, or adds or removes a companion, lands `ARCHITECTURE.md`'s index line and section range, the diff --git a/modules/dasLLAMA/REVIEW_PLACEMENT.md b/modules/dasLLAMA/REVIEW_PLACEMENT.md index af0417ff4b..11dd3652da 100644 --- a/modules/dasLLAMA/REVIEW_PLACEMENT.md +++ b/modules/dasLLAMA/REVIEW_PLACEMENT.md @@ -1,7 +1,8 @@ # dasLLAMA Code Review Checklist - placement **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -docs: the `ARCHITECTURE_*.md` set beside this file - sec.1 in each is the per-file charters. +doc: `ARCHITECTURE.md`, the index of the `ARCHITECTURE_*.md` set beside this file - sec.1 in each +companion is the per-file charters. Planned work: `followup_general.md`. **Routed from `REVIEW.md`: a diff that checklist routes here applies this list together with it.** @@ -40,13 +41,16 @@ a template declared elsewhere is not a kernel body: it compiles its own PSO wher file, never sideways into a sibling.** **A piece two files both execute lands in their nearest shared file (its own file when none -exists) - never a second copy.** A predicate, a constant, or a helper spelled once in each of -two files drifts on the first edit to one copy; an enum-and-int twin of one predicate inside one -file is the tier's idiom, not a copy, and a test's CPU oracle that restates the arithmetic is a -witness, not a copy. A piece two folders outside each other both need lands in the folder that -owns the concern; one landing under `dasllama/` that code outside `modules/dasLLAMA/` drives -lands as a public entry module - one `dasllama/dasllama_lint.das` licenses a consumer to -require directly. +exists) - never a second copy: two spellings that can drift apart on the first edit to one.** An +enum-and-int twin of one predicate inside one file is the tier's idiom, and a test's CPU oracle +that restates the arithmetic is a witness - neither is a copy. + +**A caller never re-checks a guard its callee checks - drop the caller's copy and let the +callee's check stand.** + +**A piece two folders outside each other both need lands in the folder that owns the concern; +one landing under `dasllama/` that code outside `modules/dasLLAMA/` drives lands as a public +entry module** - one `dasllama/dasllama_lint.das` licenses a consumer to require directly. **A family gaining an arm for a media kind adds that kind's span markers to that family's chat template, never to a second renderer.** Span markers are the template text that opens and @@ -87,9 +91,8 @@ module to require. A program root (test, harness, benchmark, tool) requires the module it needs directly. **A `dasllama/` module whose `[init]` registers a hook the engine dispatches through gets its -side-effect require in the same change that adds it - in `dasllama/dasllama_transformer.das`, -or in `dasllama/dasllama_common.das` where the rule above seats it there** - a registration -neither file reaches never fires for a consumer of the `dasllama.das` facade. +side-effect require in the same change that adds it** - a registration no engine file reaches +never fires for a consumer of the `dasllama.das` facade; where it lands is the rule above's. **An architecture file (`dasllama/dasllama_arch_*.das`) that changes a forward loop, or tests a family name on a shared path, is a defect - it carries declarative registration only.** diff --git a/modules/dasLLAMA/REVIEW_TTS.md b/modules/dasLLAMA/REVIEW_TTS.md index e4e66570d3..e0863c9d31 100644 --- a/modules/dasLLAMA/REVIEW_TTS.md +++ b/modules/dasLLAMA/REVIEW_TTS.md @@ -7,17 +7,19 @@ docs: `ARCHITECTURE_TTS.md`, `ARCHITECTURE_POCKET.md`. Planned work: `followup_g `REVIEW.md`.** **A family's synthesis entry point (`styletts2_synthesize`, `pocket_synthesize`) carries -`[hot_path]`, and every model stage it drives - a rows kernel in -`dasllama/dasllama_tts_blocks.das`, the assembly in a family file (`dasllama/dasllama_styletts2.das`, -`dasllama/dasllama_pocket.das`), never the text front end - sizes every buffer through a -`@scratch` carrier so the annotation holds through it.** +`[hot_path]`.** -**A buffer reused across syntheses in a file this checklist routes that is not `@scratch` - -on its declaration, or on the callee parameter it grows through - is a defect.** A `nolint` -where the annotation fits is a defect. +**A text front-end stage (`dasllama/dasllama_textnorm.das`, `dasllama/dasllama_postag.das`, +`dasllama/dasllama_g2p.das`) called below a family's synthesis entry point is a defect - +phonemize before the entry point.** + +**A buffer reused across syntheses, or filled at load for syntheses to reuse, in a file this +checklist routes that is not `@scratch` - on its declaration, or on the callee parameter it +grows through - is a defect.** The annotation is what lets `[hot_path]` hold through every stage +the entry point drives. **A function that exists for debugging or profiling, in a file this checklist routes, that is -not `[cold_path]` is a defect.** A `nolint` where `[cold_path]` fits is a defect. +not `[cold_path]` is a defect.** **A GEMM in `dasllama/dasllama_styletts2.das` or a TTS family file that does not go through a kernel `dasllama/dasllama_tts_blocks.das` exports is a defect, hand-written dot-product @@ -33,25 +35,25 @@ checked against. is a defect.** How a rows kernel stays split-invariant is the "Two layouts, one oracle" section of `ARCHITECTURE_TTS.md`. -**A new rows kernel that dispatches its rows (`maybe_parallel_for` / `lanes_for_work`) ships -its `tests/test_tts_blocks.das` bit-equality cell on both axes that move the split - the batch -lane cap and the jobque worker limit - in the same change.** - -**A new serial rows kernel ships a `tests/test_tts_blocks.das` value cell in the same change, -against the leaf it applies per row or a double-precision form of its arithmetic.** +**A new arithmetic path in `dasllama/dasllama_tts_blocks.das` - a kernel, a weight lane of one, +a layout - ships a `tests/test_tts_blocks.das` numeric cell in the same change, against the leaf +it applies per row or a double-precision form of its arithmetic; a path whose rows split across +workers, wherever the split happens - its own `maybe_parallel_for` / `lanes_for_work`, or a +backend kernel it hands a row block to - also ships the bit-equality cell on both axes that move +the split, the batch lane cap and the jobque worker limit.** **A `read_*` call in `dasllama/dasllama_styletts2.das` that leaves a conv or linear on the channel-major default while the forward assembly runs it through a rows kernel is a defect - pass the consumer (`rows`, `rows_only`) so `conv1d_prepare` / `linear_prepare` drop the layout nobody reads.** -**A caller that pins the TTS weight lane (`set_tts_q8` / `set_styletts2_q8`) around a load -resets it (`reset_tts_q8` / `reset_styletts2_q8`) before returning, on every path out, panics -included - pin through `defer()` - and pins in the context that loads: a `new_thread` context -starts every module global at its declared default, so a worker that wants a lane pins where -it loads, never through the context that spawned it.** A pin that outlives its load silently -changes the lane of the next model loaded in the process; a pin set in another context never -arrives. +**A caller that pins a TTS weight lane (`set_tts_q8` / `set_styletts2_q8` / `set_pocket_q8`) +around a load resets it (`reset_tts_q8` / `reset_styletts2_q8` / `reset_pocket_q8`) before +returning, on every path out, panics included - pin through `defer()` - and pins in the context +that loads: a `new_thread` context starts every module global at its declared default, so a +worker that wants a lane pins where it loads, never through the context that spawned it.** A pin +that outlives its load silently changes the lane of the next model loaded in the process; a pin +set in another context never arrives. **A diff that reorders the float operations of `sine_source` or `source_resize` (`dasllama/dasllama_tts_blocks.das`), or changes the rounding of any step in the phase they @@ -76,8 +78,9 @@ field reads back zero from a mapped image. moves a phoneme of the rig corpus (the corpus-identity cell in `tests/test_tts_g2p.das` decides; an unmoved corpus pins the audio bit for bit), ships the WER and UTMOS of `harness/tts_rig.py`, before and after, on every model the change reaches, on every weight -lane that model serves (`--q8`, `--f32`), at the rig's voice, in the PR body.** A lane's -per-frame figures against the f32 oracle say nothing about the speech; only the rig does. +lane that model can take - the unpinned default and each pin - at the rig's voice, in the PR +body.** A lane's per-frame figures against the f32 oracle say nothing about the speech; only +the rig does. **A text normalization or grapheme-to-phoneme error `harness/tts_rig.py`'s transcripts expose lands as a failing-first case in `tests/test_tts_textnorm.das` or @@ -94,8 +97,9 @@ them conv state, a symmetric pad, or a trim of the output by hand is a defect.** a chunk in one shot (`ARCHITECTURE_POCKET.md`, "The codec runs a chunk in one shot"); `harness/pocket_oracle.py` checks the one-shot decode against the package's frame-by-frame output. -**A change to which Pocket tensors the published file stores as Q8_0, or to their layout -(`q8_linear` / `q8_conv` in `harness/convert_pocket.py`, `read_linear` / `read_conv_q8` in -`dasllama/dasllama_pocket.das`), ships both sides in the same diff, and weakening -`test_pocket_q8_file` in `tests/test_tts_pocket.das` is a defect** - the reader's eligibility -rule and the converter's are the same rule written twice. +**A change to which quant format a published file stores a Pocket tensor in, or to its layout +(`q8_linear` / `q8_conv` / `kq_tensor` / `head_q8_linear` in `harness/convert_pocket.py`, +`read_linear` / `read_conv_q8` and the K-quant branch in `dasllama/dasllama_pocket.das`), ships +both sides in the same diff, and weakening `test_pocket_q8_file` or `test_pocket_kq_file` in +`tests/test_tts_pocket.das` is a defect** - the reader's eligibility rule and the converter's are +the same rule written twice. diff --git a/modules/dasLLAMA/REVIEW_UPSTREAM.md b/modules/dasLLAMA/REVIEW_UPSTREAM.md index 347ec753b1..9bfe21e0fc 100644 --- a/modules/dasLLAMA/REVIEW_UPSTREAM.md +++ b/modules/dasLLAMA/REVIEW_UPSTREAM.md @@ -1,4 +1,4 @@ -# dasLLAMA Upstream-Naming Code Review Checklist +# dasLLAMA Reference-Build Naming Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `ARCHITECTURE_MEASUREMENT.md`. Planned work: `followup_general.md`, `followup_vulkan.md`, @@ -8,9 +8,10 @@ doc: `ARCHITECTURE_MEASUREMENT.md`. Planned work: `followup_general.md`, `follow it.** The reference build is the third-party engine this module measures itself against - the checkout `benchmarks/setup_lcpp_ref.das` pins. Reference-build work is locating a site in that build, patching it, running it, regenerating from it, or measuring against it - planned or -performed. The text this list binds is checked-in text under `modules/dasLLAMA/`; a document -anywhere else in the tree that carries a reference-build name is bound by these same rules, -routed here by the checklist covering its own folder. +performed; a library the build ships is part of it, and running that library to mint our own +artifact is reference-build work. The text this list binds is checked-in text under +`modules/dasLLAMA/`; a document anywhere else in the tree that carries a reference-build name is +bound by these same rules, routed here by the checklist covering its own folder. **A sentence whose job is not reference-build work describes an upstream mechanism in our own terms: no "lifted/ported verbatim from", and no name belonging to the reference build - symbol, diff --git a/modules/dasLLAMA/dasllama/dasllama_env.das b/modules/dasLLAMA/dasllama/dasllama_env.das index 3acada1f87..f6728ea976 100644 --- a/modules/dasLLAMA/dasllama/dasllama_env.das +++ b/modules/dasLLAMA/dasllama/dasllama_env.das @@ -698,7 +698,7 @@ struct public TestEnv { whisper_dir : string = "" @clarg_env = "DISPLAY" - @clarg_doc = "Ambient platform variable; read only by the browser examples' smoke tests (storyteller, storywish) to tell whether a Linux box has a window server for the example's window." + @clarg_doc = "Ambient platform variable; read only by the browser examples' smoke tests (storyteller, storywish, parrot) to tell whether a Linux box has a window server for the example's window." display : string = "" @clarg_env = "WAYLAND_DISPLAY" diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index 8d22f581a9..22e6f3188f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -1547,8 +1547,8 @@ def gguf_transcode_q8_0(m : GGUFMeta; srcbytes : array | #; name : string if (src_off + expect_n > m.tensors[ti].n_elem) { panic("gguf: tensor '{name}' slice [{src_off}, {src_off + expect_n}) exceeds {m.tensors[ti].n_elem} elems") } - if (expect_n % 32l != 0l) { - panic("gguf: tensor '{name}' asks for {expect_n} Q8_0 elements, not a whole number of 32-blocks") + if (src_off % 32l != 0l || expect_n % 32l != 0l) { + panic("gguf: tensor '{name}' Q8_0 slice [{src_off}, +{expect_n}) is not block-aligned") } let nb = expect_n / 32l if (nb <= 0l) { @@ -1601,9 +1601,6 @@ def gguf_transcode_q5_0_to_q8(m : GGUFMeta; srcbytes : array | #; name : if (src_off % 32l != 0l || expect_n % 32l != 0l) { panic("gguf: tensor '{name}' Q5_0 slice [{src_off}, +{expect_n}) is not block-aligned") } - if (expect_n % 32l != 0l) { - panic("gguf: tensor '{name}' asks for {expect_n} Q8_0 elements, not a whole number of 32-blocks") - } let nb = expect_n / 32l if (nb <= 0l) { return diff --git a/modules/dasLLAMA/dasllama/dasllama_pocket.das b/modules/dasLLAMA/dasllama/dasllama_pocket.das index c2901a2afe..e375084135 100644 --- a/modules/dasLLAMA/dasllama/dasllama_pocket.das +++ b/modules/dasLLAMA/dasllama/dasllama_pocket.das @@ -114,8 +114,10 @@ struct PocketModel { bos_before_voice : bool default_voice : string voice_names : array - voices : table> // the roster's clips, 24 kHz mono + voices : table> // a voice as a clip, 24 kHz mono: a cloned one, or a roster of the older file form + voice_latents : table> // a voice as its clip's latent frames [frames][latent_dim]: the roster's stored form voice_states : table + cloning : bool // the file carries the codec encoder, so a clip can become a voice text_emb : TtsWeight = TtsWeight() // [n_bins + 1][d] bos_emb : TtsWeight = TtsWeight() // [latent_dim]: the first frame's input bos_voice : TtsWeight = TtsWeight() // [d]: the row before the voice prompt @@ -136,7 +138,8 @@ struct PocketModel { } // ===== the serving lane: f32 planes (the parity rail's form) or Q8_0 quants on the GEMMs, the -// same pin and policy as the StyleTTS2 families' ===== +// same pin and policy as the StyleTTS2 families'; unpinned, a K-quant tensor of the file serves +// as its own planes (the kq lane), so a file's formats are the lanes it can take ===== enum private PocketLane { unset @@ -152,9 +155,9 @@ def private pocket_serve_q8_() : bool { return true } -//! Pin the GEMM weights' format for subsequent loads: Q8_0 quants or the file's own f32. -//! ``reset_pocket_q8`` returns to the policy default. The facade spells these ``set_tts_q8`` / -//! ``reset_tts_q8`` for every family at once. +//! Pin the GEMM weights' format for subsequent loads: Q8_0 quants or the file's own f32 (a +//! K-quant tensor dequantizes into either); ``reset_pocket_q8`` returns to the policy default, +//! which serves a K-quant tensor as its own planes. The facade: ``set_tts_q8`` / ``reset_tts_q8``. def set_pocket_q8(on : bool) { g_pocket_q8_pin = on ? PocketLane.q8 : PocketLane.exact } @@ -166,13 +169,18 @@ def reset_pocket_q8() { //! Would the next load serve its GEMMs as q8 - the pin when set, the policy otherwise. def pocket_serves_q8() : bool => pocket_serve_q8_() +//! Would the next load serve a K-quant tensor as its own planes - true unless a lane is pinned. +def pocket_serves_native() : bool => g_pocket_q8_pin == PocketLane.unset + // what the readers mint while a load runs: the lane's quant choice and the backend's repack var private g_stage_q8 = false var private g_stage_repack = false +var private g_stage_native = false def finalize(var m : PocketModel) { delete m.voice_names delete m.voices + delete m.voice_latents delete m.voice_states delete m.text_emb delete m.bos_emb @@ -287,8 +295,8 @@ def private read_weight(m : GGUFMeta; bytes : array | #; name : string) : var private g_stage_file_q8 = false // the file's `pocket.weights` says q8: its Q8_0 tensors are in the kernels' layout // a Q8_0 tensor is read as the kernels' own blocks only where the converter wrote it that way: -// `pocket.weights = "q8"` names the layout, and the block-home rule (32-wide on both dims, a -// dense stride-1 conv) is what the converter's q8_linear / q8_conv wrote a second time +// `pocket.weights = "q8"` names the layout, and the block-home rule (32-wide on both dims, a dense +// stride-1 conv) is what q8_linear / q8_conv / head_q8_linear wrote a second time; Q4_K's disk form is ggml's own and needs no gate def private is_q8_tensor(m : GGUFMeta; name : string) : bool { if (gguf_tensor_type(m, name) != GGML_TYPE_Q8_0) { return false @@ -299,6 +307,7 @@ def private is_q8_tensor(m : GGUFMeta; name : string) : bool { return true } +[arch(at = "../ARCHITECTURE_POCKET.md#pocket-q8-file")] def private read_linear(m : GGUFMeta; bytes : array | #; prefix : string; vec_only : bool = false) : TtsLinear { var l = TtsLinear() let wname = "{prefix}.weight" @@ -308,10 +317,22 @@ def private read_linear(m : GGUFMeta; bytes : array | #; prefix : string; if (gguf_find_tensor(m, "{prefix}.bias") >= 0) { l.b <- read_arr(m, bytes, "{prefix}.bias") } - if (is_q8_tensor(m, wname) && (vec_only || l.nin % 32l != 0l || l.nout % 32l != 0l)) { - panic("dasLLAMA pocket: '{wname}' [{l.nout}][{l.nin}] is Q8_0, a shape the q8 lane does not serve ({vec_only ? "a vector-only layer" : "not 32-wide on both dims"})") + if (is_q8_tensor(m, wname) && (l.nin % 32l != 0l || l.nout % 32l != 0l)) { + panic("dasLLAMA pocket: '{wname}' [{l.nout}][{l.nin}] is Q8_0, a shape the q8 lane does not serve (not 32-wide on both dims)") + } + if (g_stage_native && gguf_tensor_type(m, wname) == GGML_TYPE_Q4_K) { + // the kq lane: the file's own K-quant planes, straight into the layer (the width rule is the layer's own) + let n = l.nout * l.nin + var inscope kq : array + var inscope ks : array + kq |> reserve_resize(n / 256l * kq_qsb(KqFmt.k4)) + ks |> reserve_resize(n / 256l * kq_ssb(KqFmt.k4)) + gguf_transcode_q4k(m, bytes, wname, kq, ks, 0l, n) + linear_take_kq(l, 4, kq, ks, true) // repacked wherever the backend the load selected carries kq kernels + return <- l } - if (g_stage_q8 && !vec_only && is_q8_tensor(m, wname)) { + if (g_stage_q8 && is_q8_tensor(m, wname)) { + // a vector-only layer the file stores Q8_0 (the flow head) takes the blocks too: its GEMV runs on the q8 lane let n = l.nout * l.nin l.wq |> reserve_resize(n) l.wqs |> reserve_resize(n / 32l) @@ -322,8 +343,8 @@ def private read_linear(m : GGUFMeta; bytes : array | #; prefix : string; l.q8 = true return <- l } - l.w <- read_arr(m, bytes, wname) // a Q8_0 tensor dequantizes here: the f32 lane of a published file - linear_prepare(l, true, g_stage_q8 && !vec_only, g_stage_repack, vec_only) + l.w <- read_arr(m, bytes, wname) // a Q8_0 or K-quant tensor dequantizes here: the f32 lane of a published file, or the q8 lane pinned over a K-quant one + linear_prepare(l, true, g_stage_q8, g_stage_repack, vec_only) // a vector layer mints nothing and keeps w return <- l } @@ -495,30 +516,36 @@ def private read_head(m : GGUFMeta; bytes : array | #; var hd : PocketHea } } -def private read_mimi(m : GGUFMeta; bytes : array | #; var mm : PocketMimi; n_layers, d, heads, ffn, context : int64; period : float) { +def private read_mimi(m : GGUFMeta; bytes : array | #; var mm : PocketMimi; n_layers, d, heads, ffn, context : int64; period : float; encoder : bool) { mm.ratios <- gguf_int_array(m, bytes, "pocket.mimi.ratios") let nr = long_length(mm.ratios) mm.hop = 1l for (r in mm.ratios) { mm.hop *= r } - // the encoder's stages read the ratios reversed: 1 -> 64 channels first, doubling per stage - mm.enc_in <- read_conv(m, bytes, "mimi.encoder.model.0.conv", 1l, 1l, false) - mm.enc_res |> resize(int(nr)) - mm.enc_down |> resize(int(nr)) - for (i in range64(nr)) { - let ratio = mm.ratios[nr - 1l - i] - mm.enc_res[i] <- read_res_conv(m, bytes, "mimi.encoder.model.{1l + 3l * i}") - mm.enc_down[i] <- read_conv(m, bytes, "mimi.encoder.model.{3l + 3l * i}.conv", ratio, 1l, false) - } - mm.enc_out <- read_conv(m, bytes, "mimi.encoder.model.{2l + 3l * nr}.conv", 1l, 1l, false) - mm.enc_tf <- read_transformer(m, bytes, "mimi.enc_tf", n_layers, d, heads, ffn, context, period, true) - mm.downsample <- read_conv(m, bytes, "mimi.downsample.conv.conv", 0l, 1l, false) - mm.frame_steps = mm.downsample.k / 2l - mm.downsample.stride = mm.frame_steps - mm.downsample.pad_l = 0l // replicate-padded by the caller, `frame_steps` rows mm.quant_proj <- read_conv(m, bytes, "mimi.quantizer.output_proj", 1l, 1l, false) + // the two resamplers are kernel 2 x stride and mirror each other; the upsampler is always in the file + mm.frame_steps = m.tensors[need_tensor(m, "mimi.upsample.convtr.convtr.weight")].dims[0] / 2l mm.upsample <- read_conv(m, bytes, "mimi.upsample.convtr.convtr", mm.frame_steps, d, true) + if (encoder) { + // the encoder's stages read the ratios reversed: 1 -> 64 channels first, doubling per stage + mm.enc_in <- read_conv(m, bytes, "mimi.encoder.model.0.conv", 1l, 1l, false) + mm.enc_res |> resize(int(nr)) + mm.enc_down |> resize(int(nr)) + for (i in range64(nr)) { + let ratio = mm.ratios[nr - 1l - i] + mm.enc_res[i] <- read_res_conv(m, bytes, "mimi.encoder.model.{1l + 3l * i}") + mm.enc_down[i] <- read_conv(m, bytes, "mimi.encoder.model.{3l + 3l * i}.conv", ratio, 1l, false) + } + mm.enc_out <- read_conv(m, bytes, "mimi.encoder.model.{2l + 3l * nr}.conv", 1l, 1l, false) + mm.enc_tf <- read_transformer(m, bytes, "mimi.enc_tf", n_layers, d, heads, ffn, context, period, true) + mm.downsample <- read_conv(m, bytes, "mimi.downsample.conv.conv", 0l, 1l, false) + if (mm.downsample.k / 2l != mm.frame_steps) { + panic("dasLLAMA pocket: the downsampler's kernel {mm.downsample.k} and the upsampler's {2l * mm.frame_steps} disagree on the frame stride") + } + mm.downsample.stride = mm.frame_steps + mm.downsample.pad_l = 0l // replicate-padded by the caller, `frame_steps` rows + } mm.dec_tf <- read_transformer(m, bytes, "mimi.dec_tf", n_layers, d, heads, ffn, context, period, true) mm.dec_in <- read_conv(m, bytes, "mimi.decoder.model.0.conv", 1l, 1l, false) mm.dec_up |> resize(int(nr)) @@ -537,14 +564,44 @@ def private lang_code_of(language : string) : string { return language } +// the roster: each voice as its stored latent frames, or as a clip in the older file form (which +// needs the encoder to hear it) +def private read_roster(g : GGUFMeta; bytes : array | #; var m : PocketModel; path : string) { + m.voice_names <- gguf_str_array(g, bytes, "pocket.voices") + for (v in m.voice_names) { + let ti = gguf_find_tensor(g, "voice_latents.{v}") + if (ti >= 0) { + if (g.tensors[ti].dims[0] != m.latent_dim) { + panic("dasLLAMA pocket: '{path}' stores voice '{v}' as frames of {g.tensors[ti].dims[0]}, the codec's latent width is {m.latent_dim}") + } + m.voice_latents[v] <- read_arr(g, bytes, "voice_latents.{v}") + } elif (m.cloning) { + m.voices[v] <- read_arr(g, bytes, "voice.{v}") + } else { + panic("dasLLAMA pocket: '{path}' stores voice '{v}' as a clip but carries no codec encoder to hear it with") + } + } +} + +//! Whether `name` is a voice this model speaks: a roster voice (stored as latent frames, or as +//! a clip in the older file form) or a cloned one. +def pocket_has_voice(m : PocketModel; name : string) : bool { + return key_exists(m.voice_latents, name) || key_exists(m.voices, name) +} + //! Load a converted Pocket TTS GGUF: the scalars from its `pocket.*` metadata, the weights -//! prepared for the rows kernels, the tokenizer, and the roster's clips (encoded on first use). +//! prepared for the rows kernels, the tokenizer, and the roster (its states built on first use). def load_pocket(path : string) : PocketModel { var inscope m = PocketModel() m.q8 = pocket_serve_q8_() g_stage_q8 = m.q8 - g_stage_repack = m.q8 && select_matmul_backend_for_load_() - to_log(LOG_INFO, "dasLLAMA pocket: GEMM lane {m.q8 ? "q8" : "f32"} - {g_pocket_q8_pin != PocketLane.unset ? "pinned via set_tts_q8" : "the policy default"}\n") + g_stage_native = pocket_serves_native() + // the backend is selected for any quantized lane; its q8 layout may want a repack (g_stage_repack), and the + // K-quant planes repack wherever it carries kq kernels - linear_take_kq asks that itself + let selected = (m.q8 || g_stage_native) ? select_matmul_backend_for_load_() : false + g_stage_repack = m.q8 && selected + let kq_arm = !g_stage_native ? "" : (kernel_backend_has_kq() ? ", its K-quant planes repacked for {active_kernel_backend()}" : ", its K-quant planes in disk order (the portable GEMV)") + to_log(LOG_INFO, "dasLLAMA pocket: GEMM lane {m.q8 ? "q8" : "f32"}{g_stage_native ? ", a K-quant tensor as its own planes" : ""}{kq_arm} - {g_pocket_q8_pin != PocketLane.unset ? "pinned via set_tts_q8" : "the policy default"}\n") let f = fopen(path, "rb") if (f == null) { panic("dasLLAMA pocket: cannot open model '{path}'") @@ -572,10 +629,8 @@ def load_pocket(path : string) : PocketModel { if (g_stage_file_q8 && !m.q8) { to_log(LOG_INFO, "dasLLAMA pocket: '{path}' carries Q8_0 GEMM weights; the f32 lane serves them dequantized\n") } - m.voice_names <- gguf_str_array(g, bytes, "pocket.voices") - for (v in m.voice_names) { - m.voices[v] <- read_arr(g, bytes, "voice.{v}") - } + m.cloning = !gguf_has(g, "pocket.cloning") || gguf_int(g, bytes, "pocket.cloning") != 0l + read_roster(g, bytes, m, path) let d = gguf_int(g, bytes, "pocket.backbone.d_model") let heads = gguf_int(g, bytes, "pocket.backbone.heads") let layers = gguf_int(g, bytes, "pocket.backbone.layers") @@ -597,7 +652,7 @@ def load_pocket(path : string) : PocketModel { read_head(g, bytes, m.head, gguf_int(g, bytes, "pocket.head.depth")) read_mimi(g, bytes, m.mimi, gguf_int(g, bytes, "pocket.mimi.layers"), gguf_int(g, bytes, "pocket.mimi.d_model"), gguf_int(g, bytes, "pocket.mimi.heads"), gguf_int(g, bytes, "pocket.mimi.dim_feedforward"), - gguf_int(g, bytes, "pocket.mimi.context"), gguf_f32(g, bytes, "pocket.mimi.rope_max_period")) + gguf_int(g, bytes, "pocket.mimi.context"), gguf_f32(g, bytes, "pocket.mimi.rope_max_period"), m.cloning) m.frame_samples = m.mimi.hop * m.mimi.frame_steps m.unk_id = gguf_int(g, bytes, "tokenizer.ggml.unknown_token_id") m.byte_fallback = gguf_int(g, bytes, "tokenizer.ggml.byte_fallback") != 0l @@ -610,6 +665,7 @@ def load_pocket(path : string) : PocketModel { } fclose(f) g_stage_file_q8 = false + g_stage_native = false m.tok <- load_tokenizer_gguf(path) return <- m } @@ -705,6 +761,9 @@ def private res_conv_rows(b : PocketResConv; x : array; t : int64; var sc //! The clip's latent frames [frames][latent_dim] through the codec encoder - the voice prompt's //! source. `pcm` is 24 kHz mono; its tail pads to a whole frame with zeros as the reference does. def pocket_encode_latents(m : PocketModel; pcm : array; var sc : PocketScratch; @scratch @exact_size var latents : array) : int64 { + if (!m.cloning) { + panic("dasLLAMA pocket: this file carries no codec encoder, so it cannot encode a clip") + } let mm & = unsafe(m.mimi) let n = long_length(pcm) let frames = (n + m.frame_samples - 1l) / m.frame_samples @@ -766,9 +825,15 @@ let POCKET_MAX_VOICE_SECONDS = 60l // the longest clip a voice is cloned from: //! front, run through the backbone at positions 0..; the state a synthesis continues from. [arch(at="../ARCHITECTURE_POCKET.md#pocket-voice-state")] def pocket_voice_state(m : PocketModel; pcm : array; var sc : PocketScratch) : PocketVoiceState { - var inscope vs = PocketVoiceState() var inscope lat : array let frames = pocket_encode_latents(m, pcm, sc, lat) + return <- voice_state_from_latents(m, lat, frames, sc) +} + +//! The voice state from a clip's latent frames [frames][latent_dim] - the stored roster form, +//! and the second half of the clip path: the speaker projection, the BOS row, the backbone. +def voice_state_from_latents(m : PocketModel; lat : array; frames : int64; var sc : PocketScratch) : PocketVoiceState { + var inscope vs = PocketVoiceState() let d = m.backbone.d let bos = m.bos_before_voice ? 1l : 0l var inscope rows : array @@ -791,6 +856,9 @@ def pocket_voice_state(m : PocketModel; pcm : array; var sc : PocketScrat //! `name` once its state is built (a clip the encoder refuses leaves no half voice behind), and a //! replaced voice's state is freed rather than left in the table's slot. def pocket_register_voice(var m : PocketModel; name : string; pcm : array; var sc : PocketScratch) { + if (!m.cloning) { + panic("dasLLAMA pocket: voice '{name}': this file carries no codec encoder, so it cannot clone a voice") + } if (empty(pcm)) { panic("dasLLAMA pocket: voice '{name}' has no samples") } @@ -804,16 +872,33 @@ def pocket_register_voice(var m : PocketModel; name : string; pcm : array } m.voice_states |> erase(name) } - if (!key_exists(m.voices, name)) { + if (!key_exists(m.voices, name) && !key_exists(m.voice_latents, name)) { m.voice_names |> push(name) } + if (key_exists(m.voice_latents, name)) { + m.voice_latents |> get(name) $(var old) { + delete old // erase frees the slot, never the frames + } + m.voice_latents |> erase(name) + } m.voices[name] := pcm m.voice_states[name] <- vs } -// a roster voice's state, encoded on first use +// a roster voice's state, built on first use: from its stored latent frames, or from its clip def private ensure_voice_state(var m : PocketModel; name : string; var sc : PocketScratch) { return if (key_exists(m.voice_states, name)) + if (key_exists(m.voice_latents, name)) { + var inscope lat : array + m.voice_latents |> get(name) $(stored) { + lat := stored + } + if (long_length(lat) % m.latent_dim != 0l) { + panic("dasLLAMA pocket: voice '{name}' stores {long_length(lat)} latent values, not whole frames of {m.latent_dim}") + } + m.voice_states[name] <- voice_state_from_latents(m, lat, long_length(lat) / m.latent_dim, sc) + return + } if (!key_exists(m.voices, name)) { panic("dasLLAMA pocket: unknown voice '{name}'") } diff --git a/modules/dasLLAMA/dasllama/dasllama_tts.das b/modules/dasLLAMA/dasllama/dasllama_tts.das index 6f79360e11..98f11141c7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tts.das +++ b/modules/dasLLAMA/dasllama/dasllama_tts.das @@ -163,7 +163,7 @@ def caps(m : TtsModel) : TtsCaps { //! What the loaded model offers: the canonical voice names the front end can drive, each with //! its language (a voice whose language it does not phonemize is left out; aliases resolve at //! synthesis), the PCM rate, the languages, whether it clones a voice, whether a speed applies. - var c = TtsCaps(sample_rate = sample_rate_of(m), cloning = m.kind == TtsKind.pocket, speed = m.kind != TtsKind.pocket) + var c = TtsCaps(sample_rate = sample_rate_of(m), cloning = m.kind == TtsKind.pocket && m.pocket.cloning, speed = m.kind != TtsKind.pocket) c.langs <- front_end_langs(m) let names & = unsafe(m.kind == TtsKind.pocket ? m.pocket.voice_names : m.model.voice_names) for (v in names) { @@ -440,7 +440,7 @@ def private synthesize_chunk(var m : TtsModel; norm : string; vname : string; sp def private resolve_voice(m : TtsModel; voice : string) : string { var vname = "" if (m.kind == TtsKind.pocket) { - vname = key_exists(m.pocket.voices, voice) ? voice : "" + vname = pocket_has_voice(m.pocket, voice) ? voice : "" } elif (m.kind == TtsKind.kitten) { vname = kitten_voice(m.model.kitten, m.model, voice) } elif (styletts2_has_voice(m.model, voice)) { diff --git a/modules/dasLLAMA/dasllama/dasllama_tts_blocks.das b/modules/dasLLAMA/dasllama/dasllama_tts_blocks.das index 9dde515a2c..2e68a80888 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tts_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_tts_blocks.das @@ -8,6 +8,7 @@ require dasllama/dasllama_lint public require math require daslib/archive require dasllama/dasllama_math +require dasllama/dasllama_math_default // matmul_kq: the portable K-quant GEMV over disk-order planes require dasllama/dasllama_par require dasllama/dasllama_plane require dasllama/dasllama_convert @@ -134,6 +135,51 @@ def weight_slot_q8(var io : TtsBlobIo; var a : array; var span : TtsSpan) } } +//! `release_weight` for a byte plane (a K-quant quant or scale plane). +def release_weight_u8(var a : array) { + if (lock_count(a) != 0) { + unsafe { + _builtin_forget_temp_array(a) + } + } else { + delete a + } +} + +//! `blob_push_q8` for a byte plane: it rides the int8 quant blob byte for byte. +def blob_push_u8(var qblob : array; src : array) : TtsSpan { + let n = long_length(src) + return TtsSpan() if (n == 0l) + let off = (long_length(qblob) + BLOB_ALIGN_FLOATS - 1l) / BLOB_ALIGN_FLOATS * BLOB_ALIGN_FLOATS + qblob |> ensure_capacity(off + n) + qblob |> resize(off + n) + for (i in range64(n)) { + qblob[off + i] = int8(src[i]) + } + return TtsSpan(off = off, n = n) +} + +//! `bind_span_q8` for a byte plane over the int8 quant plane. +def bind_span_u8(qplane : PlaneI8; span : TtsSpan; var a : array) { + return if (span.n == 0l) + if (span.off < 0l || span.n > qplane.n || span.off > qplane.n - span.n) { + panic("dasLLAMA tts: kq span {span.off}+{span.n} lies outside the {qplane.n}-byte quant blob") + } + unsafe { + _builtin_make_temp_array_i64(a, reinterpret(plane_at(qplane, span.off)), span.n) + } +} + +//! `weight_slot` for a byte plane. +def weight_slot_u8(var io : TtsBlobIo; var a : array; var span : TtsSpan) { + if (io.staging) { + span = blob_push_u8(*io.qblob, a) + delete a + } else { + bind_span_u8(io.qplane, span, a) + } +} + //! A bare weight vector (an embedding table, a Snake alpha, the ISTFT window) with its span. struct TtsWeight { a : array @@ -144,7 +190,7 @@ struct TtsWeight { // overload; the counts below are the tripwires that catch a field added without a line. let private TTS_WEIGHT_META_FIELDS = 1 let private TTS_CONV1D_META_FIELDS = 21 -let private TTS_LINEAR_META_FIELDS = 8 +let private TTS_LINEAR_META_FIELDS = 12 let private TTS_NORM_META_FIELDS = 2 let private TTS_LSTM_META_FIELDS = 5 @@ -247,15 +293,21 @@ struct TtsLinear { wt : array wq : array // [nout][nin] Q8_0 quants, repacked for the backend - the q8 rows form's W wqs : array + kq : array // [nout][nin] as a K-quant plane pair (the file's own format), repacked for a backend that carries kq + ks : array b : array w_span : TtsSpan = TtsSpan() wt_span : TtsSpan = TtsSpan() wq_span : TtsSpan = TtsSpan() wqs_span : TtsSpan = TtsSpan() + kq_span : TtsSpan = TtsSpan() + ks_span : TtsSpan = TtsSpan() b_span : TtsSpan = TtsSpan() nout : int64 nin : int64 q8 : bool + kq_fmt : int // the kernel-layer format id of the kq planes (4 = Q4_K), 0 = the layer has none + kq_repacked : bool // the planes are in the active backend's layout (matmul_kq_active), else disk order (matmul_kq) } def serialize(var arch : Archive; var l : TtsLinear) { @@ -264,10 +316,14 @@ def serialize(var arch : Archive; var l : TtsLinear) { arch |> serialize(l.wt_span) arch |> serialize(l.wq_span) arch |> serialize(l.wqs_span) + arch |> serialize(l.kq_span) + arch |> serialize(l.ks_span) arch |> serialize(l.b_span) arch |> serialize_raw(l.nout) arch |> serialize_raw(l.nin) arch |> serialize_raw(l.q8) + arch |> serialize_raw(l.kq_fmt) + arch |> serialize_raw(l.kq_repacked) } def finalize(var l : TtsLinear) { @@ -275,6 +331,8 @@ def finalize(var l : TtsLinear) { release_weight(l.wt) release_weight_q8(l.wq) release_weight(l.wqs) + release_weight_u8(l.kq) + release_weight_u8(l.ks) release_weight(l.b) } @@ -283,6 +341,8 @@ def weights_walk(var io : TtsBlobIo; var l : TtsLinear) { weight_slot(io, l.wt, l.wt_span) weight_slot_q8(io, l.wq, l.wq_span) weight_slot(io, l.wqs, l.wqs_span) + weight_slot_u8(io, l.kq, l.kq_span) + weight_slot_u8(io, l.ks, l.ks_span) weight_slot(io, l.b, l.b_span) } @@ -1052,7 +1112,7 @@ def conv1d_rows_transposed_depthwise(c : TtsConv1d; x : array; t_in : int //! Mint the tiled GEMM's B operand `wt` [nin][nout] when the width sits on the 16-column tile; //! a layer only `linear_rows` reads (`rows_only`) drops `w`, one with `q8` and per-32 widths takes -//! the Q8_0 form (`repack` for the backend), and `vec_only` - `linear_vec` reads `w` - mints none. +//! the Q8_0 form (`repack` for the backend), and `vec_only` mints none (`linear_vec` reads `w`, or the quant planes a reader handed the layer instead). def linear_prepare(var l : TtsLinear; rows_only : bool = false; q8 : bool = false; repack : bool = false; vec_only : bool = false) { return if (vec_only) @@ -1064,7 +1124,7 @@ def linear_prepare(var l : TtsLinear; rows_only : bool = false; q8 : bool = fals repack_q8q8_weight(l.wq, l.wqs, 0l, l.nin, l.nout) } l.q8 = true - delete l.w + release_weight(l.w) return } return if (l.nout % 16l != 0l) @@ -1101,23 +1161,100 @@ def private linear_rows_add_bias(l : TtsLinear; t : int64; var y : array) let nout = l.nout var yp = unsafe(addr(y[0])) let b4 = unsafe(addr(l.b[0])) + let bp = unsafe(addr(l.b[0])) maybe_parallel_for(0, int(t), lanes_for_work(t * nout, 0)) $(rb, re) { unsafe { let n4 = nout / 4l for (r in range64(int64(rb), int64(re))) { - var y4 = reinterpret(yp + r * nout) + var yrow = yp + r * nout + var y4 = reinterpret(yrow) for (j in range64(n4)) { y4[j] += b4[j] } + for (o in range64(n4 * 4l, nout)) { // the tail past the last float4: a width off 4 + yrow[o] += bp[o] + } } } } } -//! y [1][nout] = x [1][nin] . w^T + b for one row - a decode step: on the q8 lane the GEMV entry -//! (`matmul_q8q8`) instead of a one-row batch, else `linear_rows` at t = 1. The continuous-audio -//! family's frame loop calls this; the phoneme families' rows GEMMs never take it. +// ===== the K-quant lane: a layer whose weights are the file's own K-quant planes ===== + +var @scratch g_kq_xq : array +var @scratch g_kq_xs : array +var @scratch g_kq_xbs : array + +//! Give a layer the K-quant planes of its [nout][nin] weight (`fmt` the kernel-layer id, 4 = +//! Q4_K; the planes as the GGUF transcoder wrote them) and repack them for the active backend +//! where it carries kq kernels; nin is a whole number of 256-superblocks by the format's rule. +def linear_take_kq(var l : TtsLinear; fmt : int; var kq, ks : array; repack : bool) { + if (l.nin % 256l != 0l) { + panic("dasLLAMA tts: a K-quant layer [{l.nout}][{l.nin}] needs nin on 256") + } + l.kq <- kq + l.ks <- ks + l.kq_fmt = fmt + l.kq_repacked = false + if (repack && kernel_backend_has_kq()) { + repack_kq_weight(fmt, l.kq, l.ks, 0l, l.nin, l.nout) + l.kq_repacked = true + } + release_weight(l.w) +} + +// one row [nin] through the K-quant GEMV: the row requantized to the Q8_K form where it sits, +// then the backend's core over repacked planes or the portable one over disk order +def private kq_gemv_row(l : TtsLinear; x : array; xoff : int64; var y : array; yoff : int64) { + let nin = l.nin + var xq & = g_kq_xq + var xs & = g_kq_xs + var xbs & = g_kq_xbs + xq |> reserve_resize(nin) + xs |> reserve_resize(nin / 256l) + xbs |> reserve_resize(nin / 16l) + unsafe { + quantize_q8_k_into_ptr(addr(x[xoff]), nin, addr(xq[0]), addr(xs[0]), addr(xbs[0]), 0l, 0l, 0l) + } + if (l.kq_repacked) { + matmul_kq_active(l.kq_fmt, y, l.kq, l.ks, 0l, xq, xs, xbs, nin, l.nout, yoff) + } else { + matmul_kq(l.kq_fmt, y, l.kq, l.ks, 0l, xq, xs, xbs, nin, l.nout, yoff) + } +} + +// The K-quant lane of linear_rows: the rows requantized to the Q8_K form at once and one batched +// kq GEMM where the backend carries the tile, else the GEMV per row; then the bias. +[arch(at = "../ARCHITECTURE_POCKET.md#pocket-q8-file")] +def private linear_rows_kq(l : TtsLinear; x : array; t : int64; var y : array) { + let nin = l.nin + if (l.kq_repacked && kernel_backend_has_kq_batch()) { + var xq & = g_kq_xq + var xs & = g_kq_xs + var xbs & = g_kq_xbs + xq |> reserve_resize(t * nin) + xs |> reserve_resize(t * nin / 256l) + xbs |> reserve_resize(t * nin / 16l) + requant_rows_q8k_bs(x, nin, t, xq, xs, xbs, t * nin >= g_requant_par_threshold) + matmul_kq_batch(l.kq_fmt, y, l.kq, l.ks, 0l, xq, xs, xbs, nin, l.nout, t) + } else { + for (r in range64(t)) { + kq_gemv_row(l, x, r * nin, y, r * l.nout) + } + } + linear_rows_add_bias(l, t, y) +} + +//! y [1][nout] = x [1][nin] . w^T + b for one row - a decode step: the q8 lane's GEMV entry +//! (`matmul_q8q8`) or the K-quant lane's, never a one-row batch; else `linear_rows` at t = 1. +//! The continuous-audio family's frame loop calls this; the phoneme families' rows GEMMs never take it. def linear_rows_decode(l : TtsLinear; x : array; @scratch @exact_size var y : array) { + if (l.kq_fmt != 0) { + y |> reserve_resize(l.nout) + kq_gemv_row(l, x, 0l, y, 0l) + linear_rows_add_bias(l, 1l, y) + return + } if (!l.q8) { linear_rows(l, x, 1l, y) return @@ -1134,9 +1271,14 @@ def linear_rows_decode(l : TtsLinear; x : array; @scratch @exact_size var } //! y [t][nout] = x [t][nin] . w^T + b: the tiled GEMM over row blocks when `wt` was minted, -//! the batched dot form otherwise; a layer minted q8 runs the q8 lane. +//! the batched dot form otherwise; a layer minted q8 runs the q8 lane, a layer holding the +//! file's K-quant planes the kq lane. def linear_rows(l : TtsLinear; x : array; t : int64; @scratch @exact_size var y : array) { y |> reserve_resize(t * l.nout) + if (l.kq_fmt != 0) { + linear_rows_kq(l, x, t, y) + return + } if (l.q8) { linear_rows_q8(l, x, t, y) return @@ -1179,6 +1321,10 @@ def linear_rows(l : TtsLinear; x : array; t : int64; @scratch @exact_size //! y [nout] = w . x + b def linear_vec(l : TtsLinear; x : array; @scratch @exact_size var y : array) { + if (l.q8 || l.kq_fmt != 0) { // a vector layer the file stores quantized takes the decode entry's GEMV + linear_rows_decode(l, x, y) + return + } y |> reserve_resize(l.nout) matmul(y, l.w, x, l.nin, l.nout) return if (empty(l.b)) diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 82fb6b47bf..c7a4a54795 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -1420,20 +1420,26 @@ itself shows `reinterpret(13)`, an `int` widened to a pointer - a const node's whole vec4f is zero so it happens to work; that example wants a same-size spelling once the rule lands. -124. **Pocket TTS at 4 bits - part 2 of the Pocket arc (ruled 2026-09-09).** The q8 lane held - the reference's quality on the rig (alba, 200 sentences: WER 3.91 / UTMOS 4.328 on the - published Q8_0 file against the package's 5.00 / 4.393, the f32 lane at 4.32 / 4.366), and - that margin is the reason to expect a 4-bit lane to hold too. Try the engine's 4-bit weight - formats on the same GEMMs the q8 lane quantizes - the backbone's four matrices per layer, - the codec transformers, the 32-wide codec convs - through the kq plane machinery the LLM - prefill already runs (`matmul_kq_batch` over a Q8_K-requantized activation row block; - `dasllama_kqformat.das` names the formats: Q4_0, Q4_K, IQ4_NL, IQ4_XS and the rest): a - `wkq` plane beside `wq` on `TtsLinear` / `TtsConv1d`, `linear_rows_kq` and a - `conv1d_rows_dense_kq` over the same stacked tap rows, the decode step on the kq GEMV, the - published file as the winning format. One format at a time, each a rig row on both - lanes, the flow head left f32 throughout (it is the graph's sensitive part - a 1e-5 - epsilon in its timestep norm moved every latent one percent). The prize: the English file - from 152 MB to about 80, and the backbone's per-frame read from 75 MB to 38. +124. **Pocket TTS below Q4_K - what the small-format ladder left open.** The K-quant lane + is built (`TtsLinear` kq/ks planes, `linear_rows_kq` over `matmul_kq_batch`, the decode + step on `matmul_kq`; `convert_pocket.py --kq` writes the recipe as real tensors, `--fake` + scores any format through the existing lanes with no kernel behind it). The ladder that + settled the recipe, on the rig (alba, 200 sentences, the q8 file's WER 3.91 / UTMOS 4.328): + backbone Q6_K 4.09 / 4.327, Q4_K 3.91 / 4.295, Q4_0 3.73 / 4.284, IQ4_XS 4.36 / 4.326 with + seven percent more audio, Q3_K 4.50 / 4.205 (the cliff); on the Q4_K backbone the head at + Q8_0 4.09 / 4.281 and at Q4_K 3.91 / 4.259; the codec transformers at Q4_K 3.68 / 4.309; + the embedding table at Q4_K 4.00 / 4.262; the strided codec convolutions at Q8_0 3.86 / + 4.257 and at Q4_0 3.73 / 4.127 - the one rung the waveform side refuses. The recipe: + backbone Q4_K, flow head Q8_0, codec transformers Q4_K, strided codec convolutions Q8_0, + embedding Q4_K (the real file 3.86 / 4.295 on the native lane; a listen test of three + sentences in two voices against the q8 file heard no difference). The English file went + from 152 MB to 75 with the encoder and 19 voices, 65 with one voice and no encoder. Left + open, for a build that must be smaller still (the game embedding): the backbone at Q4_0 or + IQ4_XS costs 0.2 WER for the same bytes as Q4_K, so it only pays with a kernel that is + faster on the target; the head at Q4_K sits at the edge of the bar and wants the ear, not + the rig; the served 32-wide codec convolutions carry no K-quant block (width 32) and stay + Q8_0; the f16 projections and norms are untested at 8 bits. The rig row per rung and the + listen test are the gate, as before. 125. **A voice-clip upload route on dasllama-server (ruled 2026-09-09 as a ledger row).** The Pocket arc clones by NAMED voices only: the GGUF roster plus the clips of `tts_voices_dir`, read once at boot (`register_voice_clips` in `utils/dasllama-server/openai_server.das`). A @@ -1457,3 +1463,20 @@ whisper-large-v3-turbo, the language forced) is the scorer with no new tooling - a `--asr whisper` arm on the rig and one native sentence set per language (`tests/_tts_fixtures/pocket_sentences.json` has three each; the rig wants 50-200). +128. **Ogg Opus in `load_audio_mono`.** The clip loader (`dasllama/dasllama_audio_io.das`) + decodes through miniaudio, whose Ogg arm is Vorbis (stb_vorbis); an Ogg Opus file - what a + phone or a browser records as `.ogg` today - decodes to nothing, and the server's + `--tts-voices-dir` help and README say "ogg" without the distinction. Either an Opus decoder + behind the same call (libopus + the Ogg framing, a build dependency the module does not + carry yet) or the help text naming Vorbis; a clip that decodes to nothing is logged and + skipped either way. +129. **The K-quant lane's first-synthesis step.** On `pocket-tts-en-kq.gguf` the process grows + 0.33 GB at its first synthesis and holds it; on `pocket-tts-en-q8.gguf` it grows nothing + (`PERF_LEDGER.md`, the small-form section: 2.70 GB against 2.54 over the run, the load itself + lighter by 0.16). A game embedding pays that step for a 75 MB file. Where to read: the + jobque forks' persistent heaps under the rows kernels (`linear_rows_kq` over + `matmul_kq_batch`, the parallel arm of `requant_rows_q8k_bs`), the `@scratch` rows of + `dasllama/dasllama_tts_blocks.das` sized by the first chunk, and what the load leaves behind + on the K-quant path (the f32 dequant `read_linear` hands `linear_take_kq`, released per tensor + but sized by the largest). The instrument is the resident set sampled per half second with the + `--limit` one and two forms, and the das leak profiler on the run. diff --git a/modules/dasLLAMA/harness/convert_pocket.py b/modules/dasLLAMA/harness/convert_pocket.py index d0a99f6cca..d94e232263 100644 --- a/modules/dasLLAMA/harness/convert_pocket.py +++ b/modules/dasLLAMA/harness/convert_pocket.py @@ -15,9 +15,12 @@ - the unigram SentencePiece tokenizer as `tokenizer.ggml.model = "t5"` (upstream's name for a unigram model) with `tokenizer.ggml.tokens` / `scores` / `token_type` and the special ids; - the model's scalars as `pocket.*` metadata (from the config, not guessed); -- each bundled voice clip as `voice.` [samples] f32 PCM at 24 kHz mono - the language's - roster; the CC BY-NC clips of the package's English roster are left out (the sidecar names - every clip's source and licence). +- each bundled voice as `voice_latents.` [frames][latent_dim] f32 - the clip's frames + through the package's own codec encoder, the form a voice state is built from (the reader + also takes the older `voice.` PCM form, which needs the encoder); `--voices` picks the + roster, the CC BY-NC clips of the package's English roster are never in it (the sidecar names + every clip's source and licence); `--no-cloning` leaves the codec encoder out and says so in + `pocket.cloning`, so the file serves its roster and refuses to clone. Canonical names: `flow_lm.transformer.layers.N.*` -> `backbone.N.*`; `flow_lm.flow_net.*` -> `head.*`; `mimi.encoder_transformer.transformer.layers.N.*` -> `mimi.enc_tf.N.*`, the decoder @@ -83,8 +86,9 @@ def q8_linear(name, shape): """The linears the engine serves through the rows GEMM as Q8_0: the transformer layers' - four matrices and the frame input projection, on widths that quantize per 32. The head's - GEMVs, the EOS head, the speaker projection and the norms stay f16.""" + four matrices and the frame input projection, on widths that quantize per 32. The EOS head, + the speaker projection and the norms stay f16; the flow head's GEMVs stay f16 in the --q8 + form and go Q8_0 in the --kq form (head_q8_linear).""" rows_served = (".self_attn." in name or ".linear1." in name or ".linear2." in name) and name.endswith(".weight") \ and (name.startswith("backbone.") or name.startswith("mimi.enc_tf.") or name.startswith("mimi.dec_tf.")) rows_served = rows_served or name == "flow_lm.input_linear.weight" @@ -100,6 +104,135 @@ def q8_conv(name, shape, stride, transposed): return is_conv and len(shape) == 3 and not transposed and stride == 1 and shape[0] % 32 == 0 and shape[1] % 32 == 0 +FAKE_GROUPS = ("attn", "ffn", "input", "speaker", "embed", "head", "codec", "codecconv", "strided") + + +def fake_group(name, shape, conv_served=False): + """The tensor group a `--fake` spec names: the backbone's attention projections, its two FFN + matrices, the frame input projection, the speaker projection, the text embedding table, the + flow head's matrices, the codec transformers' GEMMs (`codec`), the convs the engine serves + q8 (`codecconv`), and the codec's strided, transposed and resampling convs the file keeps + f16 (`strided`). Norms, biases and the voices are never in a group.""" + if name == "flow_lm.speaker_proj_weight": # the one matrix the bundle names without the ".weight" suffix + return "speaker" if len(shape) == 2 else None + if not name.endswith(".weight") or len(shape) < 2: + return None + if name.startswith("backbone."): + if ".self_attn." in name: + return "attn" + if ".linear1." in name or ".linear2." in name: + return "ffn" + return None + if name == "flow_lm.input_linear.weight": + return "input" + if name == "flow_lm.conditioner.embed.weight": + return "embed" + if name.startswith("head."): + return "head" + if name.startswith("mimi."): + if len(shape) == 3: + return "codecconv" if conv_served else "strided" + return "codec" + return None + + +def parse_fake(spec): + """`group=fmt,group=fmt` - a format per group, ggml's names (q4_0, q4_k, q6_k, iq4_nl, ...).""" + out = {} + for item in filter(None, spec.split(",")): + assert item.count("=") == 1, f"--fake wants group=fmt, got '{item}'" + group, fmt = item.split("=") + assert group in FAKE_GROUPS, (group, FAKE_GROUPS) + assert group not in out, f"--fake names {group} twice" + out[group] = fmt.upper() + return out + + +class FakeQuant: + """Round a float matrix through a ggml quant format and back: ggml's own quantizer (the + built llama.cpp's libggml-base) writes the blocks, gguf-py reads them back to f32. The rows + then store in the file's usual form, so the engine's lanes measure the format's loss with no + new kernel; a width the format's block does not divide is left as it is and reported.""" + + def __init__(self, llama_cpp, gguf_mod): + import ctypes + import glob + libs = glob.glob(os.path.join(llama_cpp, "build", "bin", "libggml-base.dylib")) + \ + glob.glob(os.path.join(llama_cpp, "build", "bin", "libggml-base.so")) + assert libs, "no built libggml-base beside llama.cpp/build/bin - build llama.cpp first" + self.lib = ctypes.CDLL(libs[0]) + self.lib.ggml_quantize_chunk.restype = ctypes.c_size_t + self.lib.ggml_quantize_chunk.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_float), ctypes.c_void_p, + ctypes.c_int64, ctypes.c_int64, ctypes.c_int64, ctypes.POINTER(ctypes.c_float)] + self.ctypes = ctypes + self.gguf = gguf_mod + self.skipped = [] + self.done = {} + + def quantize_bytes(self, name, rows, fmt): + """`rows` [nrows][n_per_row] f32 -> the format's blocks, [nrows][bytes per row] u8 (None + where the width is not a whole number of blocks).""" + t = self.gguf.GGMLQuantizationType[fmt] + block, type_size = self.gguf.GGML_QUANT_SIZES[t] + nrows, n_per_row = rows.shape + if n_per_row % block != 0: + self.skipped.append((name, fmt, n_per_row)) + return None + out = np.empty((nrows, (n_per_row // block) * type_size), dtype=np.uint8) + c = self.ctypes + n = self.lib.ggml_quantize_chunk(int(t), rows.ctypes.data_as(c.POINTER(c.c_float)), out.ctypes.data_as(c.c_void_p), + 0, nrows, n_per_row, None) + assert n == out.nbytes, (name, fmt, n, out.nbytes) + return out + + def apply(self, name, w32, fmt, group): + from gguf.quants import dequantize + t = self.gguf.GGMLQuantizationType[fmt] + # a conv rounds per slice of its leading dim over the rest ([cout][cin][k] per output + # channel; a transposed conv's [cin][cout][k] per input channel); a matrix per row + rows = np.ascontiguousarray(w32.reshape(w32.shape[0], -1) if w32.ndim == 3 else w32.reshape(-1, w32.shape[-1]), dtype=np.float32) + out = self.quantize_bytes(name, rows, fmt) + if out is None: + return w32 + back = dequantize(out, t).reshape(w32.shape).astype(np.float32) + err = float(np.sqrt(((back - w32) ** 2).mean()) / max(np.sqrt((w32 ** 2).mean()), 1e-12)) + self.done[name] = (group, fmt, out.nbytes, err) + return back + + +def encoder_tensor(name): + """The codec encoder: the SEANet stages, the encoder transformer and the frame downsampler - + everything a clip goes through on its way to latents, and nothing a synthesis reads.""" + return name.startswith("mimi.encoder.") or name.startswith("mimi.enc_tf.") or name.startswith("mimi.downsample.") + + +def clip_encoder(language): + """The package's own codec encoder over a 24 kHz clip -> its latent frames [frames][latent_dim], + the form the roster is stored in: a voice is the backbone's memory of those frames, so the + file needs no clip samples and, with --no-cloning, no encoder.""" + import torch + from pocket_tts import TTSModel + model = TTSModel.load_model(language=language) + + def encode(pcm): + with torch.no_grad(): + lat = model.mimi.encode_to_latent(torch.from_numpy(np.ascontiguousarray(pcm, dtype=np.float32))[None, None]) + return np.ascontiguousarray(lat[0].numpy(), dtype=np.float32) + return encode + + +def kq_tensor(name, shape): + """The tensors `--kq` stores as Q4_K: the backbone's and the codec transformers' matrices and + the text embedding table - every one 256-wide along its rows, the K-quant rule.""" + return fake_group(name, shape) in ("attn", "ffn", "codec", "embed") and len(shape) == 2 and shape[1] % 256 == 0 + + +def head_q8_linear(name, shape): + """The flow head's matrices, which `--kq` stores as Q8_0: the vector layers' GEMVs run on the + q8 lane where the file holds the blocks.""" + return name.startswith("head.") and name.endswith(".weight") and len(shape) == 2 and shape[0] % 32 == 0 and shape[1] % 32 == 0 + + def canonical(name): if name.startswith("flow_lm.transformer.layers."): return "backbone." + name[len("flow_lm.transformer.layers."):] @@ -169,10 +302,22 @@ def main(): ap.add_argument("--llama-cpp", default=os.path.expanduser("~/Work/llama.cpp"), help="for gguf-py") ap.add_argument("--name", default=None, help="output file stem (default pocket-tts-)") ap.add_argument("--q8", action="store_true", help="the published form: the served GEMM weights as Q8_0 in the kernels' layout") + ap.add_argument("--fake", default="", help=f"quality experiment, never published: group=fmt[,group=fmt] (groups {', '.join(FAKE_GROUPS)}; " + "ggml format names) - the group's weights round through that format before they are stored; needs --name") + ap.add_argument("--kq", action="store_true", help="the small form: the backbone's and the codec transformers' matrices and the text embedding " + "as Q4_K, the flow head as Q8_0, the rest as --q8 writes it (implies --q8)") + ap.add_argument("--voices", default="", help="the roster as a comma list of the language's voice names (default: every voice the language ships)") + ap.add_argument("--no-cloning", action="store_true", help="leave the codec encoder out: the roster speaks from its stored latents, " + "tts_register_voice refuses, and the file is smaller by the encoder") a = ap.parse_args() sys.path.insert(0, os.path.join(a.llama_cpp, "gguf-py")) import gguf from gguf.quants import quantize as gguf_quantize + fake = parse_fake(a.fake) + assert not fake or a.name, "--fake files are local experiments: give them a --name" + if a.kq: + a.q8 = True + fq = FakeQuant(a.llama_cpp, gguf) if (fake or a.kq) else None lang = a.language cfg = load_config(lang) @@ -192,11 +337,22 @@ def main(): conv_stride["mimi.downsample.conv.conv.weight"] = st["mimi.downsample.conv.conv.weight"].shape[2] // 2 # kernel 2 x stride tensors = {} quantized = [] + quantized_k4 = [] + dropped = [] for k, v in st.items(): name = canonical(k) assert len(name) < GGML_MAX_NAME, name assert name not in tensors, name - if a.q8 and q8_linear(name, v.shape): + if a.no_cloning and encoder_tensor(name): + dropped.append(name) + continue + group = fake_group(name, v.shape, q8_conv(name, v.shape, conv_stride.get(name, 1), ".convtr." in name)) if fake else None + if group in fake: + v = fq.apply(name, np.ascontiguousarray(v.astype(np.float32)), fake[group], group) + if a.kq and kq_tensor(name, v.shape): + tensors[name] = ("k4", np.ascontiguousarray(v.astype(np.float32))) + quantized_k4.append(name) + elif a.q8 and (q8_linear(name, v.shape) or (a.kq and head_q8_linear(name, v.shape))): tensors[name] = ("q8", np.ascontiguousarray(v.astype(np.float32))) quantized.append(name) elif a.q8 and q8_conv(name, v.shape, conv_stride.get(name, 1), ".convtr." in name): @@ -209,6 +365,12 @@ def main(): voices = {} sources = {} roster = dict(ENGLISH_VOICES) if lang.startswith("english") else LANGUAGE_VOICES[lang] + if a.voices: + picked = [v.strip() for v in a.voices.split(",") if v.strip()] + unknown = [v for v in picked if v not in roster] + assert not unknown, f"--voices names {unknown}; the {lang} roster is {sorted(roster)}" + roster = {v: roster[v] for v in picked} + encoder = clip_encoder(lang) for vname, (rel, licence) in roster.items(): if rel.startswith("tts-voices:"): path = os.path.join(a.hub, "tts-voices", rel[len("tts-voices:"):]) @@ -220,10 +382,13 @@ def main(): print(f" voice {vname}: {path} missing - skipped", flush=True) continue pcm = read_clip(path) - voices[vname] = pcm + latents = encoder(pcm) + voices[vname] = latents sources[vname] = (rel, licence, len(pcm)) - tensors["voice." + vname] = pcm + tensors["voice_latents." + vname] = latents default_voice = DEFAULT_VOICE.get(lang, next(iter(voices))) + if default_voice not in voices: + default_voice = next(iter(voices)) assert default_voice in voices, (default_voice, list(voices)) fl = cfg["flow_lm"] @@ -231,8 +396,13 @@ def main(): stem = a.name or (f"pocket-tts-{lang}-q8" if a.q8 else f"pocket-tts-{lang}") path = os.path.join(a.out, stem + ".gguf") w = gguf.GGUFWriter(path, ARCH) - w.add_name(f"Pocket TTS {lang}" + (" Q8_0" if a.q8 else "")) + w.add_name(f"Pocket TTS {lang}" + (" Q4_K" if a.kq else " Q8_0" if a.q8 else "")) w.add_string("pocket.weights", "q8" if a.q8 else "f16") + if a.kq: + w.add_string("pocket.kq", "q4_k") + w.add_bool("pocket.cloning", not a.no_cloning) + if fake: + w.add_string("pocket.fake", a.fake) w.add_string("pocket.language", lang) w.add_string("pocket.revision", weights_rev) w.add_string("pocket.tokenizer_revision", tok_rev) @@ -283,7 +453,10 @@ def main(): w.add_bool("tokenizer.ggml.byte_fallback", spec["byte_fallback"]) for name in sorted(tensors): t = tensors[name] - if isinstance(t, tuple): + if isinstance(t, tuple) and t[0] == "k4": + data = fq.quantize_bytes(name, t[1], "Q4_K") # ggml's own quantizer; the writer derives the element shape from the byte shape + w.add_tensor(name, data, raw_dtype=gguf.GGMLQuantizationType.Q4_K) + elif isinstance(t, tuple): data = gguf_quantize(t[1], gguf.GGMLQuantizationType.Q8_0) # the writer derives the element shape from the byte shape w.add_tensor(name, data, raw_dtype=gguf.GGMLQuantizationType.Q8_0) else: @@ -293,18 +466,33 @@ def main(): w.write_tensors_to_file() w.close() with open(path + ".LICENSE", "w", encoding="utf8") as f: - f.write(f"{stem}.gguf - Kyutai Pocket TTS ({lang}) weights{' (the served GEMMs as Q8_0)' if a.q8 else ''}, CC BY 4.0 (Kyutai), converted from kyutai/pocket-tts " + f.write(f"{stem}.gguf - Kyutai Pocket TTS ({lang}) weights{' (the backbone and codec transformers as Q4_K, the head as Q8_0)' if a.kq else (' (the served GEMMs as Q8_0)' if a.q8 else '')}" + f"{' without the codec encoder' if a.no_cloning else ''}, CC BY 4.0 (Kyutai), converted from kyutai/pocket-tts " f"languages/{lang}/model.safetensors @ {weights_rev} and the unigram SentencePiece tokenizer @ {tok_rev} by " "modules/dasLLAMA/harness/convert_pocket.py; the reference implementation is MIT (github.com/kyutai-labs/pocket-tts). " - "Bundled voice clips:\n") + "Bundled voices (each the codec encoder's latent frames of the clip named):\n") for vname, (rel, licence, n) in sources.items(): - f.write(f" voice.{vname}: {rel} ({n / SAMPLE_RATE:.1f} s) - {licence}\n") + f.write(f" voice_latents.{vname}: {rel} ({n / SAMPLE_RATE:.1f} s) - {licence}\n") for vname, why in EXCLUDED_VOICES.items(): if lang.startswith("english"): f.write(f" not shipped: {vname} - {why}\n") f.write("see the dasLLAMA THIRD_PARTY_NOTICES.md\n") - print(f"wrote {path}: {len(tensors)} tensors ({len(quantized)} as Q8_0), {len(pieces)} pieces, {len(voices)} voices " - f"(default {default_voice}); {os.path.getsize(path)} bytes on disk") + print(f"wrote {path}: {len(tensors)} tensors ({len(quantized)} as Q8_0{f', {len(quantized_k4)} as Q4_K' if quantized_k4 else ''}" + f"{f', {len(dropped)} encoder tensors left out' if dropped else ''}), " + f"{len(pieces)} pieces, {len(voices)} voices (default {default_voice}); {os.path.getsize(path)} bytes on disk") + if fq: + by_group = {} + for name, (group, fmt, nbytes, err) in fq.done.items(): + g = by_group.setdefault((group, fmt), [0, 0, 0.0]) + g[0] += 1 + g[1] += nbytes + g[2] = max(g[2], err) + for (group, fmt), (n, nbytes, err) in sorted(by_group.items()): + print(f" fake {group}={fmt}: {n} tensors, {nbytes / 1e6:.1f} MB as {fmt}, worst rms rel err {err:.4f}") + for name, fmt, width in fq.skipped: + print(f" fake skipped {name}: width {width} is not a whole number of {fmt} blocks") + unmatched = [group for group in fake if not any(g == group for g, _ in by_group) and not any(fake_group(n, (1, 1)) == group for n, _, _ in fq.skipped)] + assert not unmatched, f"--fake named groups no tensor belongs to: {unmatched}" if __name__ == "__main__": diff --git a/modules/dasLLAMA/harness/publish_tts_hf.py b/modules/dasLLAMA/harness/publish_tts_hf.py index 7a74714e31..df5c3ccc3a 100644 --- a/modules/dasLLAMA/harness/publish_tts_hf.py +++ b/modules/dasLLAMA/harness/publish_tts_hf.py @@ -16,7 +16,7 @@ CARD = os.path.join(HERE, "tts_model_card.md") FILES = ["kitten-nano.gguf", "kitten-mini.gguf", "kokoro-82m.gguf", "tts_g2p.bin", "tts_g2p_en_us.bin", "tts_postag.bin", "pocket-tts-en-q8.gguf", "pocket-tts-de-q8.gguf", "pocket-tts-es-q8.gguf", "pocket-tts-it-q8.gguf", - "pocket-tts-pt-q8.gguf", "pocket-tts-fr-q8.gguf"] + "pocket-tts-pt-q8.gguf", "pocket-tts-fr-q8.gguf", "pocket-tts-en-kq.gguf", "pocket-tts-en-stuart-kq.gguf"] LICENCES = ["LICENSE.APACHE-2.0", "LICENSE.CMUDICT", "LICENSE.UD_EWT", "LICENSE.SPACY", "LICENSE.STYLETTS2", "LICENSE.CC-BY-4.0", "LICENSE.POCKET_TTS"] diff --git a/modules/dasLLAMA/harness/test_convert_pocket.py b/modules/dasLLAMA/harness/test_convert_pocket.py new file mode 100644 index 0000000000..7e989f3ce0 --- /dev/null +++ b/modules/dasLLAMA/harness/test_convert_pocket.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""convert_pocket.py's pure predicates - which tensor lands in which form - with no torch, no +safetensors and no network: the groups a `--fake` spec names, the `--kq` and head rules, the +encoder set `--no-cloning` leaves out. The converted files' contents are the das cells' claim +(tests/test_tts_pocket.das); this file holds the rules that decide them.""" +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import convert_pocket as cp # noqa: E402 + +ATTN = "backbone.0.self_attn.in_proj.weight" +FFN = "backbone.3.linear1.weight" +NORM = "backbone.3.norm1.weight" +INPUT = "flow_lm.input_linear.weight" +SPEAKER = "flow_lm.speaker_proj_weight" # the one matrix the bundle names without the ".weight" suffix +EMBED = "flow_lm.conditioner.embed.weight" +HEAD = "head.blocks.1.mlp.0.weight" +HEAD_IN = "head.input_proj.weight" +CODEC_TF = "mimi.dec_tf.2.self_attn.in_proj.weight" +CODEC_CONV = "mimi.decoder.model.4.conv.weight" +STRIDED = "mimi.encoder.model.3.conv.weight" +ENC_TF = "mimi.enc_tf.0.linear2.weight" +DOWNSAMPLE = "mimi.downsample.conv.conv.weight" +VOICE = "voice_latents.alba" + + +class FakeGroupTest(unittest.TestCase): + def test_every_group_is_reached_by_the_tensor_it_names(self): + self.assertEqual(cp.fake_group(ATTN, (3072, 1024)), "attn") + self.assertEqual(cp.fake_group(FFN, (4096, 1024)), "ffn") + self.assertEqual(cp.fake_group(INPUT, (1024, 32)), "input") + self.assertEqual(cp.fake_group(SPEAKER, (1024, 512)), "speaker") + self.assertEqual(cp.fake_group(EMBED, (4000, 1024)), "embed") + self.assertEqual(cp.fake_group(HEAD, (1024, 1024)), "head") + self.assertEqual(cp.fake_group(CODEC_TF, (1536, 512)), "codec") + self.assertEqual(cp.fake_group(CODEC_CONV, (512, 512, 3), conv_served=True), "codecconv") + self.assertEqual(cp.fake_group(STRIDED, (256, 128, 8)), "strided") + for group in ("attn", "ffn", "input", "speaker", "embed", "head", "codec", "codecconv", "strided"): + self.assertIn(group, cp.FAKE_GROUPS) + + def test_what_no_group_names(self): + self.assertIsNone(cp.fake_group(NORM, (1024,)), "a norm is one-dimensional") + self.assertIsNone(cp.fake_group("backbone.0.self_attn.in_proj.bias", (3072,)), "a bias is not a weight") + self.assertIsNone(cp.fake_group("backbone.0.norm1.weight", (1024, 1)), "a backbone tensor outside attention and the FFN") + self.assertIsNone(cp.fake_group(VOICE, (120, 32)), "a voice is never in a group") + + def test_a_spec_names_a_format_per_group(self): + self.assertEqual(cp.parse_fake("attn=q4_k,head=q8_0"), {"attn": "Q4_K", "head": "Q8_0"}) + self.assertEqual(cp.parse_fake(""), {}) + with self.assertRaises(AssertionError): + cp.parse_fake("norms=q4_k") + + +class SmallFormTest(unittest.TestCase): + def test_kq_takes_the_256_wide_matrices_of_the_backbone_the_codec_transformers_and_the_embedding(self): + self.assertTrue(cp.kq_tensor(ATTN, (3072, 1024))) + self.assertTrue(cp.kq_tensor(FFN, (4096, 1024))) + self.assertTrue(cp.kq_tensor(CODEC_TF, (1536, 512))) + self.assertTrue(cp.kq_tensor(ENC_TF, (512, 2048))) + self.assertTrue(cp.kq_tensor(EMBED, (4000, 1024))) + + def test_kq_leaves_the_rest_alone(self): + self.assertFalse(cp.kq_tensor(HEAD, (1024, 1024)), "the flow head stays Q8_0") + self.assertFalse(cp.kq_tensor(INPUT, (1024, 32)), "the frame input projection is 32 wide") + self.assertFalse(cp.kq_tensor(SPEAKER, (1024, 512)), "the speaker projection") + self.assertFalse(cp.kq_tensor(ATTN, (3072, 1000)), "a width the 256-superblock does not divide") + self.assertFalse(cp.kq_tensor(CODEC_CONV, (512, 512, 3)), "a conv is three-dimensional") + + def test_the_head_goes_to_q8_on_widths_that_quantize_per_32(self): + self.assertTrue(cp.head_q8_linear(HEAD, (1024, 1024))) + self.assertTrue(cp.head_q8_linear(HEAD_IN, (1024, 32))) + self.assertFalse(cp.head_q8_linear("head.blocks.1.mlp.0.bias", (1024,))) + self.assertFalse(cp.head_q8_linear(HEAD, (1024, 30))) + self.assertFalse(cp.head_q8_linear(ATTN, (3072, 1024)), "the backbone is not the head") + + def test_no_cloning_leaves_out_exactly_the_clip_path(self): + for name in (STRIDED, ENC_TF, DOWNSAMPLE, "mimi.encoder.model.0.conv.weight"): + self.assertTrue(cp.encoder_tensor(name), name) + for name in (CODEC_TF, CODEC_CONV, "mimi.upsample.convtr.convtr.weight", "mimi.quantizer.output_proj.weight", ATTN, HEAD, VOICE): + self.assertFalse(cp.encoder_tensor(name), name) + + +class PublishedFormTest(unittest.TestCase): + def test_q8_serves_the_rows_gemms_and_the_frame_input(self): + self.assertTrue(cp.q8_linear(ATTN, (3072, 1024))) + self.assertTrue(cp.q8_linear(ENC_TF, (512, 2048))) + self.assertTrue(cp.q8_linear(INPUT, (1024, 32))) + self.assertFalse(cp.q8_linear(HEAD, (1024, 1024)), "the head is the --kq form's alone") + self.assertFalse(cp.q8_linear(SPEAKER, (1024, 512))) + self.assertFalse(cp.q8_linear(ATTN, (3072, 1000))) + + def test_q8_convs_are_the_dense_forward_stride_one_ones(self): + self.assertTrue(cp.q8_conv(CODEC_CONV, (512, 512, 3), 1, False)) + self.assertTrue(cp.q8_conv("mimi.quantizer.output_proj.weight", (512, 256, 1), 1, False)) + self.assertFalse(cp.q8_conv(STRIDED, (256, 128, 8), 4, False), "a strided stage") + self.assertFalse(cp.q8_conv("mimi.decoder.model.5.convtr.convtr.weight", (256, 128, 8), 4, True), "a transposed stage") + self.assertFalse(cp.q8_conv("mimi.decoder.model.14.conv.weight", (1, 64, 7), 1, False), "the single-channel end") + + +if __name__ == "__main__": + unittest.main() diff --git a/modules/dasLLAMA/harness/tts_model_card.md b/modules/dasLLAMA/harness/tts_model_card.md index 1bd6dbe2e8..b3d0f2e4b3 100644 --- a/modules/dasLLAMA/harness/tts_model_card.md +++ b/modules/dasLLAMA/harness/tts_model_card.md @@ -51,6 +51,8 @@ No espeak-ng, no phonemizer: the front end is data, and the data is in the two p | `pocket-tts-it-q8.gguf` | Pocket TTS Italian (6 layers), one voice (`giovanni`) | 134415072 | `3c5739d544b1b7c8284fd3df9d7122557cf700c91895d3dc45b3fdc5ef6e2670` | | `pocket-tts-pt-q8.gguf` | Pocket TTS Portuguese (6 layers), one voice (`rafael`) | 134667488 | `3375c31e742c8783c6dddbbd3bd152e8dff9d188fdaceb1cbb4d187514291c57` | | `pocket-tts-fr-q8.gguf` | Pocket TTS French (24 layers, the only French model Kyutai ships), one voice (`estelle`) | 375793696 | `f06ffac80b96a34d2e51ca40c41111469d8b44e0269b27a64e707a7a9be1ec20` | +| `pocket-tts-en-kq.gguf` | Pocket TTS English in the small form: the backbone and the codec transformers as Q4_K, the flow head and the codec convolutions as Q8_0, the embedding table Q4_K; its tokenizer, the codec encoder (so it clones) and the 19 voices as latent frames | 74970016 | `2475a1ed8d49eb72c9d9b8c38f10f91ef5b03c7cd6e9fe43fdf7ab00ae1a0a25` | +| `pocket-tts-en-stuart-kq.gguf` | the same small form with one voice (`stuart_bell`) as latent frames and no codec encoder: reads text in that voice, cannot clone | 65107520 | `bc9604b527066134354dc480e20c960f63f5c3538c1dd757ba409bd782cddac9` | The packs sit beside whichever GGUF you load; the loader reads them from the model's directory - `tts_g2p.bin` when it is there, else `tts_g2p_en_us.bin`. The GGUFs carry f32 weights: dasLLAMA quantizes the served layouts to Q8_0 at first @@ -70,12 +72,20 @@ Pocket TTS English is the cloning model: 152 MB, 19 voices (`alba` the default, front of it. On the 200-sentence rig at `alba` this file reads WER 3.91 / UTMOS 4.328 at a real-time factor of 0.051 on an Apple M1 Max, against the reference package's 5.00 / 4.393 / 0.210 (measured 2026-09-09 with the module's `harness/tts_rig.py`, the engine under the JIT -tier with the box's tune profile, the reference package under torch on one thread). The five other +tier with the box's tune profile - `DAS_TUNE_POLICY` unset - on the `arm64-gen` kernel backend, +the reference package under torch on one thread). The five other languages are the same form, one file each with Kyutai's default clip for that language as its only voice (German `juergen`, Spanish `lola`, Italian `giovanni`, Portuguese `rafael`, French `estelle`); the German, Spanish, Italian and Portuguese files are the six-layer models, French exists only as the 24-layer one. A voice cloned from any clip speaks the file's language with the clip's accent. Text in those languages is read as it is, since the normalizer is English. +`pocket-tts-en-kq.gguf` is the English model in the small form, 75 MB: the backbone and the +codec transformers as Q4_K, the flow head and the codec convolutions as Q8_0, the embedding +table Q4_K, the encoder and the 19 voices inside (on the rig at `alba`: WER 3.86 / UTMOS 4.295 +at a real-time factor of 0.044, measured 2026-09-10 on the same box, tier, tune profile and +kernel backend, the file's Q4_K planes served as they are); it is the file the browser examples on +dasllama.io fetch. `pocket-tts-en-stuart-kq.gguf` is that form with one voice, `stuart_bell`, +and no codec encoder, 65 MB: it reads text in that voice and cannot clone. Kitten nano is the phoneme families' served default: 59 MB, eight voices, a real-time factor of 0.03 on an Apple M1 Max (measured 2026-09-02 with the same rig). Its voices are `expr-voice-2-m` through @@ -109,6 +119,10 @@ voices. `kyutai/pocket-tts-without-voice-cloning` at `d29db7978e464fb90cb3359ee0c69a273b9142cc`; the voice clips from `kyutai/tts-voices` at `323332d33f997de8394f24a193e1a76df720e01a` (`voice-zero/`, `voice-donations/`, `vctk/`, `alba-mackenna/casual.wav`). +- `pocket-tts-en-kq.gguf` / `pocket-tts-en-stuart-kq.gguf`: the same sources through + `modules/dasLLAMA/harness/convert_pocket.py --kq` (the second with `--voices stuart_bell + --no-cloning`); the K-quant blocks are ggml's own quantizer, and each voice is stored as the + latent frames of its clip through the model's codec encoder. The whole set is rebuilt by `modules/dasLLAMA/performance/build_tts_data.das`. Parity against the reference implementations (block by block, and the front end sentence by sentence on a @@ -122,7 +136,7 @@ the reference implementations (block by block, and the front end sentence by sen | `kokoro-82m.gguf` | Apache-2.0 | hexgrad's weights and voices, converted; the architecture is StyleTTS2 (MIT, `LICENSE.STYLETTS2`) | | `tts_g2p.bin` | Apache-2.0 and BSD-2-Clause | misaki and g2p_en (Apache-2.0), CMUdict (`LICENSE.CMUDICT`, Carnegie Mellon University) | | `tts_postag.bin` | CC BY-SA 4.0 | the tagger weights are trained on UD English-EWT (`LICENSE.UD_EWT`); the exception table and the silver tags come from spaCy (MIT, `LICENSE.SPACY`); Gutenberg prose is public domain | -| `pocket-tts-en-q8.gguf` | CC BY 4.0 | Kyutai's weights and tokenizer, converted (`LICENSE.CC-BY-4.0`); the reference implementation is MIT (`LICENSE.POCKET_TTS`) and not included; the voice clips: `voice-zero` and `voice-donations` CC0, VCTK (CSTR, University of Edinburgh) and Alba Mackenna CC BY 4.0 - the sidecar lists each | +| `pocket-tts-*.gguf` (every Pocket file, the two `-kq` ones included) | CC BY 4.0 | Kyutai's weights and tokenizer, converted (`LICENSE.CC-BY-4.0`); the reference implementation is MIT (`LICENSE.POCKET_TTS`) and not included; the voice clips: `voice-zero` and `voice-donations` CC0, VCTK (CSTR, University of Edinburgh) and Alba Mackenna CC BY 4.0 - the sidecar lists each | Each `.LICENSE` sidecar beside a file names its sources; the full texts are in this repository. The engine that reads these files is under the daslang licence in its own repository. diff --git a/modules/dasLLAMA/performance/REVIEW.md b/modules/dasLLAMA/performance/REVIEW.md index 243975a79d..0e46ad0e03 100644 --- a/modules/dasLLAMA/performance/REVIEW.md +++ b/modules/dasLLAMA/performance/REVIEW.md @@ -109,8 +109,13 @@ or an unnamed table lookup. **A diff that adds a companion artifact - a file fetched or verified with a model and consumed beside it: a projector, a draft head, an assistant sidecar, an image fixture - puts it in the -`companions` of the row that pins its carrier, and names it from every other row that -consumes it.** +`companions` of the row that pins its carrier.** + +**A diff that adds a row whose test cell consumes a companion pinned on another row - a fixture, +an oracle, a twin file that cell reads beside it - names that companion in the new row's +`companions`, in the same change; a diff that adds a companion an existing row's cell consumes +names it from that row too.** A box that fetches the row alone gets its companions with it, and +the cell runs instead of skipping. **A diff that changes what any `serve_*` function in `model_specs.das` returns - a `serve_*` field on a row, the function's body, or a `companions` entry with a `url` on a row a `serve_*` diff --git a/modules/dasLLAMA/performance/model_specs.das b/modules/dasLLAMA/performance/model_specs.das index bc7793e6c3..298b045a13 100644 --- a/modules/dasLLAMA/performance/model_specs.das +++ b/modules/dasLLAMA/performance/model_specs.das @@ -279,11 +279,14 @@ let REF_Q8_RECIPE = "derive: setup_asr_rig.das --refs quantizes it from the fp b let PARAKEET_V2_NOTE = "historic nemo-venv conversion (rail retired); present only on boxes that converted it - v3 is the fetched carrier" let CANARY_ENC_RECIPE = "convert: modules/dasLLAMA/harness/convert_canary_to_ggml.py over nvidia/canary-qwen-2.5b (nemo venv; encoder is a repack - sha canonical everywhere)" let CANARY_DEC_RECIPE = "convert: modules/dasLLAMA/harness/convert_canary_to_ggml.py over nvidia/canary-qwen-2.5b (nemo venv), then llama-quantize Q8_0 (a Q8_0 disk embedding lets the tied classifier serve cls_q8 on the Metal rail); bytes are per-arch (fp16 LoRA merge) - gate via asr_bench --text over jfk/jfk3/gb1 vs benchmarks/asr/canary_transcripts.expected" -let TTS_HF = "{HF}/borisbat/dasllama-tts/resolve/f3d4818a01076e3f235c2f77a5ac4227ccf355b2" // the published set, commit-pinned +let TTS_HF = "{HF}/borisbat/dasllama-tts/resolve/be9630af1cba2f6f7946efa7b8ea194b9b093dc9" // the published set, commit-pinned let TTS_KITTEN_RECIPE = "convert: modules/dasLLAMA/performance/build_tts_data.das -- --root --out runs harness/convert_kitten.py over KittenML/kitten-tts-nano-0.8 @ 7a1db645b1f3ab9420761d87428e042b9cec3f26 and kitten-tts-mini-0.8 @ c02725660cea441db4c383af69f1f26f5cd00947 (the ONNX weights into GGUF, f32); THIRD_PARTY_NOTICES.md (repo root) carries the attribution" let TTS_KOKORO_RECIPE = "convert: modules/dasLLAMA/performance/build_tts_data.das -- --root --out runs harness/convert_kokoro.py over hexgrad/Kokoro-82M @ f3ff3571791e39611d31c381e3a41a3af07b4987 (kokoro-v1_0.pth + the voice packs into GGUF, f32); THIRD_PARTY_NOTICES.md (repo root) carries the attribution" let TTS_POCKET_RECIPE = "convert: modules/dasLLAMA/harness/convert_pocket.py --language english_2026-04 --q8 --name pocket-tts-en-q8 over kyutai/pocket-tts languages/english_2026-04/model.safetensors @ 19f95fe2df36e79fbd9f10008595cc4c977a0fcc, the tokenizer of kyutai/pocket-tts-without-voice-cloning @ d29db7978e464fb90cb3359ee0c69a273b9142cc and the voice clips of kyutai/tts-voices @ 323332d33f997de8394f24a193e1a76df720e01a (the served GEMMs as Q8_0, the rest f16); THIRD_PARTY_NOTICES.md (repo root) carries the attribution" let TTS_POCKET_LANG_RECIPE = "convert: modules/dasLLAMA/harness/convert_pocket.py --language --q8 --name pocket-tts--q8 over kyutai/pocket-tts languages//model.safetensors @ 39592ff23c9ef80098bb74895d104c26275fe2c9 (german, italian, spanish, portuguese, french_24l), the tokenizer of kyutai/pocket-tts-without-voice-cloning @ d29db7978e464fb90cb3359ee0c69a273b9142cc and the language's default clip (kyutai/pocket-tts @ 64ab7d24c479d736a83b8cc666c4a776fca30fda; estelle from kyutai/tts-voices @ 1fc7395b7e012e2bbebfca14b942a4ef62ccc899); THIRD_PARTY_NOTICES.md (repo root) carries the attribution" +//! the small form the browser examples fetch: K-quant planes where the ladder held (backbone and codec transformers Q4_K, the embedding table Q4_K), Q8_0 where it did not (the flow head, the codec convolutions), the roster as latent frames +let TTS_POCKET_KQ_RECIPE = "convert: modules/dasLLAMA/harness/convert_pocket.py --language english_2026-04 --kq --name pocket-tts-en-kq over the sources of pocket-tts-en-q8.gguf's recipe (the backbone and the codec transformers as Q4_K through ggml's own quantizer, the flow head and the codec convolutions as Q8_0, the embedding table Q4_K, the codec encoder inside, the 19 voices as the latent frames of their clips); THIRD_PARTY_NOTICES.md (repo root) carries the attribution" +let TTS_POCKET_STUART_RECIPE = "convert: modules/dasLLAMA/harness/convert_pocket.py --language english_2026-04 --kq --voices stuart_bell --no-cloning --name pocket-tts-en-stuart-kq over the sources of pocket-tts-en-q8.gguf's recipe (the small form of pocket-tts-en-kq.gguf with one voice as latent frames and no codec encoder: reads text in stuart_bell, cannot clone); THIRD_PARTY_NOTICES.md (repo root) carries the attribution" // the f16 twin the parity rail loads on its f32 lane: the same sources, no --q8; never published let TTS_POCKET_F16_RECIPE = "convert: modules/dasLLAMA/harness/convert_pocket.py --language english_2026-04 --name pocket-tts-en over the sources of pocket-tts-en-q8.gguf's recipe (every tensor f16; the parity rail's reference under tests/test_tts_pocket.das, local only)" let TTS_PACKS_RECIPE = "mint: modules/dasLLAMA/performance/build_tts_data.das -- --root --out runs harness/build_g2p_data.py (misaki 0.9.4 gold/silver + CMUdict 0.7a + the g2p_en 2.1.0 GRU + harness/g2p_local_additions.json into tts_g2p.bin) and harness/train_postag.py (UD English-EWT + spaCy-labelled silver prose into tts_postag.bin)" @@ -645,7 +648,15 @@ def model_specs() : array { // nolint:STYLE038 — flat model-set t url = "{TTS_HF}/pocket-tts-fr-q8.gguf", bytes = 375793696l, sha256 = "f06ffac80b96a34d2e51ca40c41111469d8b44e0269b27a64e707a7a9be1ec20", companions <- [ - ProvEntry(name = "tts_oracle/pocket_french_24l", root = "llm", recipe = TTS_ORACLE_RECIPE)]) + ProvEntry(name = "tts_oracle/pocket_french_24l", root = "llm", recipe = TTS_ORACLE_RECIPE)]), + // the small English forms behind the browser examples: the parrot page fetches the cloning one, + // the storywish page the one-voice one (tests/test_tts_pocket.das's kq cells hold both against the q8 file) + ModelSpec(file = "pocket-tts-en-kq.gguf", recipe = TTS_POCKET_KQ_RECIPE, serve_tts = true, + url = "{TTS_HF}/pocket-tts-en-kq.gguf", bytes = 74970016l, + sha256 = "2475a1ed8d49eb72c9d9b8c38f10f91ef5b03c7cd6e9fe43fdf7ab00ae1a0a25"), + ModelSpec(file = "pocket-tts-en-stuart-kq.gguf", recipe = TTS_POCKET_STUART_RECIPE, serve_tts = true, + url = "{TTS_HF}/pocket-tts-en-stuart-kq.gguf", bytes = 65107520l, + sha256 = "bc9604b527066134354dc480e20c960f63f5c3538c1dd757ba409bd782cddac9") ] } diff --git a/modules/dasLLAMA/tests/CLAUDE.md b/modules/dasLLAMA/tests/CLAUDE.md index d1175e4980..b1083bbf0e 100644 --- a/modules/dasLLAMA/tests/CLAUDE.md +++ b/modules/dasLLAMA/tests/CLAUDE.md @@ -730,7 +730,13 @@ languages (`pocket-tts-{de,es,it,pt,fr}-q8.gguf`, oracle dirs `tts_oracle/pocket minted over `_tts_fixtures/pocket_sentences.json`, token fixtures `pocket_tokens_.json`): the language code and the one default voice, the tokenizer on the language's own sentences and the probes, the teacher-forced frames of every oracle case, one sentence through the -facade. +facade. Two more carriers gate their own cells: the small form `pocket-tts-en-kq.gguf` (the +K-quant file: its Q4_K tensors arrive as kq planes on the unpinned lane, the head as Q8_0; the +teacher-forced latents of the kq lane held to the q8 lane of the same file at the q8 bar with a +poisoned-expectation control, the f16 twin's distance logged, the exact lane speaking) and the +one-voice `pocket-tts-en-stuart-kq.gguf` (no codec encoder: `caps()` reports one voice and +`cloning = false`, the stored voice speaks from its latent frames, `tts_register_voice` refuses +by name). `test_tts_facade.das` - stocked suite; model-free cells: the sentence chunker (the reference driver's boundary rule, the cap counted in codepoints, the hard split of a whitespace-free run, the appended comma as Kitten's driver rule and the bare text Kokoro's sends), the normalizer the diff --git a/modules/dasLLAMA/tests/REVIEW.md b/modules/dasLLAMA/tests/REVIEW.md index 5691816b89..123cabd201 100644 --- a/modules/dasLLAMA/tests/REVIEW.md +++ b/modules/dasLLAMA/tests/REVIEW.md @@ -51,7 +51,7 @@ a run of skips is not the coverage the suite owes. **A diff that changes what a file covers - a cell added, removed or moved, its suite, an axis or bar an existing cell asserts - corrects that file's `CLAUDE.md` census entry, numbers included, in the same change.** A `{a,b}` shorthand naming several files at once, or a suite roster, -carries nothing to correct. +carries nothing to correct; a file with no census entry owes none, a file with one keeps it true. **A diff that changes the contract a gate pins - what its asserts hold fixed, an axis gained or lost - updates that gate's entry in this checklist's pinned set in the same change.** @@ -126,7 +126,9 @@ other stocked fixture gates on its own presence. **A test - or a program a test builds or spawns - whose subject is not the `.dlim` image rail never mints or maps a MODEL image: it either runs with `DASLLAMA_IMAGE=0` in its environment, -or calls no `load_model`, `load_model_cached`, or `load_model_image`.** +or calls no loader that bakes a `.dlim` - `load_model`, `load_model_cached`, `load_model_image`, +`load__tower`, `load__encoder`, `load__model`, `load_tts_model`, +`load_styletts2`.** **A predicate whose value the BOX decides (a device capability, a policy default) and that therefore cannot differ between two runs on one machine is never tested through its own @@ -239,10 +241,10 @@ procedurally and pins its expectations in-repo.** `DASLLAMA_VISION_DUMP` cannot preview, is a defect** - a red never requires adding instrumentation before a human can see what the model consumed. -**An audio clip a test feeds an embedder that the test does not build, and that is not one of -the clips stocked beside the models (`jfk.wav`, `gemma4a_test2.wav`), is a defect** - a clip -nobody else can play makes a red unreadable. A newly stocked clip joins this list in the same -change. +**An audio clip a test feeds an embedder that the test does not build, that the repository does +not track, and that is not one of the clips stocked beside the models (`jfk.wav`, +`gemma4a_test2.wav`), is a defect** - a clip nobody else can play makes a red unreadable. A newly +stocked clip joins this list in the same change. **A media fixture an embedder-parity cell regenerates in-test and compares against an oracle dump, with no exact-value generator - one whose values are exactly representable floats, so diff --git a/modules/dasLLAMA/tests/_example_rail.das b/modules/dasLLAMA/tests/_example_rail.das index 3f2a26bb6d..b06261e266 100644 --- a/modules/dasLLAMA/tests/_example_rail.das +++ b/modules/dasLLAMA/tests/_example_rail.das @@ -6,7 +6,7 @@ require dasllama/dasllama_env require daslib/fio require strings -// Shared by the browser examples' smoke tests (test_storyteller_restart.das, test_storywish.das): +// Shared by the browser examples' smoke tests (test_storyteller_restart.das, test_storywish.das, test_parrot.das): // each spawns its example as a child of this binary, and the example opens a GLFW window. //! this binary, as the child's argv[0] diff --git a/modules/dasLLAMA/tests/run.das b/modules/dasLLAMA/tests/run.das index 5d62020198..bca4ce7d90 100644 --- a/modules/dasLLAMA/tests/run.das +++ b/modules/dasLLAMA/tests/run.das @@ -196,6 +196,7 @@ def suite_files(name : string) : array { // nolint:STYLE038 - a flat s "modules/dasLLAMA/tests/test_mtp_gemma_drafter.das", "modules/dasLLAMA/tests/test_parity.das", "modules/dasLLAMA/tests/test_parity_pregate.das", + "modules/dasLLAMA/tests/test_parrot.das", "modules/dasLLAMA/tests/test_ple_modes.das", "modules/dasLLAMA/tests/test_prefill.das", "modules/dasLLAMA/tests/test_qwen3v.das", @@ -241,13 +242,13 @@ let TESTS_DIR = "modules/dasLLAMA/tests" def area_tests(area : string) : array { if (area == "audio") { return <- [ "test_asr_verbs.das", "test_audio.das", "test_audio_embedder.das", "test_dasllama_lint_contracts.das", - "test_tower_asr_kernels.das", "test_tower_helpers.das", "test_vad.das", "test_whisper.das" ] + "test_parrot.das", "test_tower_asr_kernels.das", "test_tower_helpers.das", "test_vad.das", "test_whisper.das" ] } elif (area == "vision") { return <- [ "test_attn_span.das", "test_gemma3v.das", "test_gemma4uv.das", "test_gemma4v.das", "test_qwen25v.das", "test_qwen3v.das", "test_tower_helpers.das", "test_vision.das", "test_vision_chat.das", "test_vision_embedder.das" ] } elif (area == "tts") { - return <- [ "test_storyteller_restart.das", "test_storywish.das", "test_tts_blocks.das", "test_tts_facade.das", + return <- [ "test_parrot.das", "test_storyteller_restart.das", "test_storywish.das", "test_tts_blocks.das", "test_tts_facade.das", "test_tts_g2p.das", "test_tts_kitten.das", "test_tts_kokoro.das", "test_tts_pocket.das", "test_tts_postag.das", "test_tts_textnorm.das" ] } elif (area == "infra") { diff --git a/modules/dasLLAMA/tests/test_parrot.das b/modules/dasLLAMA/tests/test_parrot.das new file mode 100644 index 0000000000..4dba487a23 --- /dev/null +++ b/modules/dasLLAMA/tests/test_parrot.das @@ -0,0 +1,213 @@ +options gen2 +options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) +options _dasllama_internal = true + +require dastest/testing_boost public +require daslib/env_registry +require daslib/fio +require daslib/strings_boost +require strings +require math +require _model_tier +require _example_rail +require ../../../examples/dasLLAMA/parrot/take.das + +// Parrot (examples/dasLLAMA/parrot): the take's pure side (take.das) is tested directly - the +// resampler Silero hears through, the rule that ends a take, the clip's window, the text box's +// editing rules and a button's box. The smoke cell spawns the example under --smoke with the +// tree's own clip in place of the microphone and reads the witness lines it logs: the voice is +// cloned from the clip, the text is said in it, and the say is read out to the end. Model-gated +// on pocket-tts-en-kq.gguf (the Pocket file with its codec encoder, the one the page ships) under +// models_dir(); the voice-activity model is the tree's own. The run opens the example's window +// for about half a minute, so a box without a window server skips, and --null-audio keeps it +// silent - and off the microphone, which --clip never opens anyway. + +[test] +def test_vad_resampler(t : T?) { + t |> run("24 kHz to 16 kHz over the whole take: fed in drains of any size, the stream is the one fed at once") @(t : T?) { + var take : array + take |> resize(24000) + for (i in range(24000)) { + take[i] = sin(float(i) * 0.013) + 0.3 * cos(float(i) * 0.071) + } + var whole = VadResampler() + var at_once : array + resample_step(whole, take, at_once) + t |> equal(16000, length(at_once), "one second of 24 kHz is 16000 samples of 16 kHz (the last at position 23998.5, still inside the take)") + t |> equal(at_once[0], take[0], "the first sample is the take's first") + t |> success(abs(at_once[2] - take[3]) < 1.0e-6, "position 3.0 lands on the take's third sample") + t |> success(abs(at_once[1] - 0.5 * (take[1] + take[2])) < 1.0e-6, "position 1.5 is the mean of its neighbours") + var drained = VadResampler() + var stream : array + var partial : array + var fed = 0 + var chunk = 37 + while (fed < 24000) { + let n = min(chunk, 24000 - fed) + partial |> resize(fed + n) + for (i in range(fed, fed + n)) { + partial[i] = take[i] + } + fed += n + chunk = (chunk * 7 + 11) % 900 + 1 // drains of every size the device might hand over + var piece : array + resample_step(drained, partial, piece) + stream |> push_from(piece) + } + t |> equal(length(stream), length(at_once), "the same count") + var diffs = 0 + for (a, b in stream, at_once) { + diffs += a != b ? 1 : 0 + } + t |> equal(diffs, 0, "sample for sample the same stream - no seam where the drains met") + } + t |> run("a detector index maps back to the take") @(t : T?) { + t |> equal(vad_to_mic(16000l), 24000l) + t |> equal(vad_to_mic(0l), 0l) + t |> equal(vad_to_mic(512l), 768l) + } +} + +[test] +def test_take_rules(t : T?) { + t |> run("the take ends two seconds after speech, at the cap, or when the device gave nothing for long") @(t : T?) { + t |> success(!take_ends(false, false, 100.0, 10.0, 0.0), "no speech yet, nothing ends it") + t |> success(!take_ends(true, true, 100.0, 10.0, 0.0), "speech still going") + t |> success(!take_ends(true, false, 1.9, 10.0, 0.0), "quiet, not yet two seconds") + t |> success(take_ends(true, false, 2.0, 10.0, 0.0), "quiet for two seconds") + t |> success(take_ends(true, true, 0.0, 60.0, 0.0), "the cap, mid-speech") + t |> success(take_ends(false, false, 0.0, 60.0, 0.0), "the cap, no speech") + t |> success(!take_ends(false, false, 0.0, 0.0, 5.9), "a device silent for less than six seconds") + t |> success(take_ends(false, false, 0.0, 0.0, 6.0), "a device that gave nothing for six seconds") + } + t |> run("the clip's window: the speech plus a pad, an open speech to the end, never past the cap") @(t : T?) { + let pad = int64(TAKE_PAD_S * float(MIC_RATE)) + var w = clip_window(48000l, 96000l, 240000l, false) + t |> equal(w.from, 48000l - pad, "a quarter second before the speech") + t |> equal(w.to, 96000l + pad, "a quarter second after it") + w = clip_window(2000l, 96000l, 240000l, false) + t |> equal(w.from, 0l, "the pad never reaches before the take") + w = clip_window(48000l, 96000l, 100000l, false) + t |> equal(w.to, 100000l, "the pad never reaches past the take") + w = clip_window(48000l, 0l, 240000l, true) + t |> equal(w.to, 240000l, "speech still open at the stop runs to the take's end") + let cap = int64(TAKE_CAP_S) * int64(MIC_RATE) + w = clip_window(0l, 0l, cap + 24000l * 3l, true) + t |> equal(w.to - w.from, cap, "a take past the cap yields a clip of exactly the cap") + w = clip_window(240000l, 0l, 240000l, true) + t |> success(w.to >= w.from, "an empty window is never negative") + } +} + +[test] +def test_text_box(t : T?) { + t |> run("a key lands as its character: letters follow shift, the punctuation the poem needs, nothing else") @(t : T?) { + t |> equal(typed_char('A', false), 'a') + t |> equal(typed_char('A', true), 'A') + t |> equal(typed_char('1', false), '1') + t |> equal(typed_char('1', true), '!') + t |> equal(typed_char('/', true), '?') + t |> equal(typed_char(';', true), ':') + t |> equal(typed_char('\'', true), '"') + t |> equal(typed_char('-', true), '_') + t |> equal(typed_char(',', false), ',') + t |> equal(typed_char(' ', false), ' ') + t |> equal(typed_char('[', false), -1, "a bracket is not taken") + t |> equal(typed_char('=', false), -1, "nor an equals sign") + t |> equal(typed_char(300, false), -1, "nor a key past ASCII") + } + t |> run("the box: a line fills to its width, the box to its height, backspace walks back over lines") @(t : T?) { + var lines <- lines_of("ab\ncd") + t |> equal(length(lines), 2) + t |> equal(lines[1], "cd") + append_char(lines, 'e') + t |> equal(lines[1], "cde") + for (_i in range(WRAP_CHARS)) { + append_char(lines, 'x') + } + t |> equal(length(lines[1]), WRAP_CHARS, "a full line takes no more") + erase_last(lines) + t |> equal(length(lines[1]), WRAP_CHARS - 1) + for (_i in range(MAX_LINES + 5)) { + new_line(lines) + } + t |> equal(length(lines), MAX_LINES, "a full box takes no more lines") + for (_i in range(MAX_LINES)) { + erase_last(lines) + } + t |> equal(length(lines), 2, "backspace on an empty line drops it, down to the line above") + var one <- lines_of("") + t |> equal(length(one), 1, "an empty text is one empty line") + erase_last(one) + t |> equal(length(one), 1, "the only line stays") + } + t |> run("a button answers inside its label's box with a margin") @(t : T?) { + let box = float4(0.0, -20.0, 120.0, 4.0) // left, top, right, bottom relative to the pen + t |> success(inside_box(box, 60.0, 600.0, 100.0, 590.0, 8.0), "on the glyphs above the pen") + t |> success(inside_box(box, 60.0, 600.0, 55.0, 605.0, 8.0), "inside the margin") + t |> success(!inside_box(box, 60.0, 600.0, 100.0, 640.0, 8.0), "below the label") + t |> success(!inside_box(box, 60.0, 600.0, 200.0, 590.0, 8.0), "past its right edge") + } +} + +let CLIP = "modules/dasLLAMA/models/jfk_ask_not.wav" +let SMOKE_TEXT = "The woods are lovely, dark and deep. But I have promises to keep." +let CLONED_LINE = "parrot: cloned " +let SAY_LINE = "parrot: say 1 in voice you, 65 characters" +let READ_LINE = "parrot: say 1 is read out" +let DONE_LINE = "parrot: the text was read out" + +//! the window server and the model file, each absence registering its own loud skip +def private ready(t : T?) : bool { + if (!has_window_server()) { + t |> skip("no window server (DISPLAY and WAYLAND_DISPLAY unset) - the example opens a window") + return false + } + return model_available(t, path_join(models_dir(), "pocket-tts-en-kq.gguf")) +} + +//! the example's smoke rail: the clip cloned, the text said in the clone, read out to the end (the spawn's wall clock is the guard, not a frame cap) +def private run_parrot(var out : string&) : int { + let prev_image = env_value_of("DASLLAMA_IMAGE") + set_env_variable("DASLLAMA_IMAGE", "0") //! never mint a sidecar beside the stocked models + let root = get_das_root() + let argv <- [example_daslang(), "-jit", "-dasroot", root, "{root}/examples/dasLLAMA/parrot/main.das", "--", + "--models", models_dir(), "--smoke", "--clip", path_join(root, CLIP), "--text", SMOKE_TEXT, "--null-audio"] + let rc = run_and_capture(argv, out, 300.0) + set_env_variable("DASLLAMA_IMAGE", prev_image) + return rc +} + +def private witness(out : string) : string { + var inscope lines <- split(out, "\n") + return join([for (line in lines); line; where find(line, "parrot:") >= 0], "\n") +} + +def private tail_of(out : string) : string { + return slice(out, max(0, length(out) - 600)) +} + +[test] +def test_smoke_clones_and_says(t : T?) { + t |> run("the smoke rail clones the voice from the clip, says the text in it and reads it out to the end") @(t : T?) { + return if (!ready(t)) + var out = "" + let rc = run_parrot(out) + return if (skip_without_window(t, rc, out)) + t |> equal(0, rc, "the smoke run exits clean: {tail_of(out)}") + t |> success(find(out, CLONED_LINE) >= 0, "the voice was cloned from the clip:\n{witness(out)}") + t |> success(find(out, SAY_LINE) >= 0, "the text was said in the cloned voice:\n{witness(out)}") + t |> success(find(out, READ_LINE) >= 0, "the say was read out:\n{witness(out)}") + t |> success(find(out, DONE_LINE) >= 0, "the rail ended on the read-out, not the frame cap:\n{witness(out)}") + } + t |> run("a --clip that is not audio is refused by name, before any window opens") @(t : T?) { + return if (!ready(t)) + var out = "" + let root = get_das_root() + let argv <- [example_daslang(), "-jit", "-dasroot", root, "{root}/examples/dasLLAMA/parrot/main.das", "--", + "--models", models_dir(), "--smoke", "--clip", "{root}/examples/dasLLAMA/parrot/models.json", "--null-audio"] + let rc = run_and_capture(argv, out, 60.0) + t |> success(rc != 0, "a clip that does not decode is a refusal, not a run") + t |> success(find(out, "did not decode") >= 0, "and it says so by name:\n{tail_of(out)}") + } +} diff --git a/modules/dasLLAMA/tests/test_run_suites.das b/modules/dasLLAMA/tests/test_run_suites.das index a8e28a0085..d999ae43de 100644 --- a/modules/dasLLAMA/tests/test_run_suites.das +++ b/modules/dasLLAMA/tests/test_run_suites.das @@ -156,7 +156,7 @@ def test_areas_for_path(t : T?) { [test] def test_area_plan(t : T?) { let tts <- area_plan(["tts"]) - t |> equal(length(tts.files), 11, "tts plans its ten files and the image file its kitten arm rides") + t |> equal(length(tts.files), 12, "tts plans its eleven files and the image file its kitten arm rides") t |> equal(tts.image_arms, "kitten", "tts owns the kitten image arm") t |> equal(tts.files[length(tts.files) - 1], "{TESTS_DIR}/test_model_image.das", "the image file rides last") for (i in range(1, length(tts.files) - 1)) { @@ -178,7 +178,7 @@ def test_area_plan(t : T?) { t |> equal(length(suite.files), 1, "a suite run plans the suite table") t |> equal(suite.image_arms, "arm12", "a suite run passes --arm through") let area <- plan_files(cfg, ["tts"]) - t |> equal(length(area.files), 11, "an area run plans the area, whatever --suite says") + t |> equal(length(area.files), 12, "an area run plans the area, whatever --suite says") } //! the argument contracts: an area run refuses --arm/--full/--suite and an unknown area; a per-PR diff --git a/modules/dasLLAMA/tests/test_storywish.das b/modules/dasLLAMA/tests/test_storywish.das index 76ee7cb0c2..10690051c0 100644 --- a/modules/dasLLAMA/tests/test_storywish.das +++ b/modules/dasLLAMA/tests/test_storywish.das @@ -17,11 +17,11 @@ require ../../../examples/dasLLAMA/storywish/wish.das // layout, and a line the model writes is recognized as a field the corpus puts after a story. // The smoke cell spawns the example under --smoke with fixed words and reads the witness lines it // logs: the story starts with those words and is written and read out to the end. Model-gated on -// tinystories-instruct-27M-Q8_0.gguf (its ../performance/model_specs.das row names the source), -// kitten-nano.gguf and the front-end packs under models_dir(); the run opens the example's window -// for about a minute, so a box without a window server skips, and --null-audio keeps it silent. +// tinystories-instruct-27M-Q8_0.gguf (its ../performance/model_specs.das row names the source) +// and pocket-tts-en-stuart-kq.gguf (the one-voice Pocket file the page ships) under models_dir(); +// the run opens the example's window for about a minute, so a box without a window server +// skips, and --null-audio keeps it silent. -let PACK_FILES <- ["tts_g2p.bin", "tts_postag.bin"] let SMOKE_WORDS = "dragon, cake, moon" let START_LINE = "storywish: story 1 starts at frame 0 - words: dragon, cake, moon" let DONE_LINE = "storywish: story 1 written and read out" @@ -66,15 +66,9 @@ def private ready(t : T?) : bool { t |> skip("no window server (DISPLAY and WAYLAND_DISPLAY unset) - the example opens a window") return false } - for (name in ["tinystories-instruct-27M-Q8_0.gguf", "kitten-nano.gguf"]) { + for (name in ["tinystories-instruct-27M-Q8_0.gguf", "pocket-tts-en-stuart-kq.gguf"]) { return false if (!model_available(t, path_join(models_dir(), name))) } - for (name in PACK_FILES) { - if (!stat(path_join(models_dir(), name)).is_valid) { - t |> skip("{name} not present") - return false - } - } return true } diff --git a/modules/dasLLAMA/tests/test_tts_blocks.das b/modules/dasLLAMA/tests/test_tts_blocks.das index 309c9997b4..945c78cfa2 100644 --- a/modules/dasLLAMA/tests/test_tts_blocks.das +++ b/modules/dasLLAMA/tests/test_tts_blocks.das @@ -6,8 +6,14 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama_tts_blocks require dasllama/dasllama_math // setup_dasllama_jobque_ +require dasllama/dasllama_math_default // dot_k4q8, the K-quant leaf per row +require dasllama/dasllama_convert // transcode_q4k_superblock, quantize_q8_k_into_ptr +require dasllama/dasllama_gemm_schema // kq_qsb / kq_ssb by format id +require daslib/f16_cvt require daslib/jobque_boost require daslib/defer +require daslib/rtti // this_context().last_exception: a refusal read as text +require strings require math // The TTS block home's two layouts against each other: every rows-form kernel (token-major @@ -21,6 +27,7 @@ let REL = 1.0e-4 let FLOOR = 1.0e-6 let T_ROWS = 2048l let STYLE = 32l +let KQ_REL = 2.0e-4 //! the K-quant lane against its own leaf: the repacked tile sums the superblocks' scale products in its own order (the disk-order arm is bit-exact) struct BlockCase { name : string @@ -527,6 +534,133 @@ def private linear_case_q8(nin, nout : int64) : BlockCase { return res } +// ===== the K-quant lane against the leaf per row ===== + +def private put_f16(var b : array; off : int64; v : float) { + let bits = f32_to_f16(v) + b[off] = uint8(bits & 0xFFu) + b[off + 1l] = uint8(bits >> 8u) +} + +//! one Q4_K superblock in the GGUF's disk layout: two f16 scales, 12 bytes of 6-bit sub-scales, 128 bytes of nibbles +def private q4k_disk_superblock() : array { + var b : array + b |> resize(144l) + put_f16(b, 0l, 0.375) + put_f16(b, 2l, 0.125) + for (i in range64(12l)) { + b[4l + i] = uint8((i * 37l + 13l) % 256l) + } + for (i in range64(128l)) { + b[16l + i] = uint8((i * 29l + 7l) % 256l) + } + return <- b +} + +//! [rows][n] of Q4_K planes as the GGUF transcoder writes them: the one superblock everywhere, its nibbles salted per row +def private build_q4k_planes(var kq : array; var ks : array; rows, n : int64; seed : uint) { + let nsb = n / 256l + let qsb = kq_qsb(4) + let ssb = kq_ssb(4) + var inscope blk <- q4k_disk_superblock() + kq |> resize(rows * nsb * qsb) + ks |> resize(rows * nsb * ssb) + for (r in range64(rows)) { + for (s in range64(nsb)) { + transcode_q4k_superblock(blk, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } + var st = seed + uint(r) * 0x85EBCA6Bu + for (i in range64(nsb * qsb)) { + st = st * 1664525u + 1013904223u + kq[r * nsb * qsb + i] = uint8(uint(kq[r * nsb * qsb + i]) ^ (st >> 16u)) + } + } +} + +//! the leaf the lane applies per row: the row requantized to the Q8_K form, dot_k4q8 over the disk-order planes, then the bias +def private kq_leaf_rows(kq, ks : array; x : array; t, nin, nout : int64; b : array; var want : array) { + let nsb = nin / 256l + let qsb = kq_qsb(4) + let ssb = kq_ssb(4) + var xq : array + var xs : array + var xbs : array + xq |> resize(nin) + xs |> resize(nsb) + xbs |> resize(nin / 16l) + want |> resize(t * nout) + unsafe { + for (p in range64(t)) { + quantize_q8_k_into_ptr(addr(x[p * nin]), nin, addr(xq[0]), addr(xs[0]), addr(xbs[0]), 0l, 0l, 0l) + for (o in range64(nout)) { + let d = dot_k4q8(addr(kq[o * nsb * qsb]), addr(ks[o * nsb * ssb]), + addr(xq[0]), addr(xs[0]), addr(xbs[0]), nin) + want[p * nout + o] = d + b[o] + } + } + } +} + +// linear_rows (the batched kq GEMM where the backend carries the tile, the GEMV per row otherwise) +// and linear_rows_decode (the GEMV) on the K-quant lane against the leaf per row; `repack` false +// holds the planes in disk order - the portable arm every tier without kq kernels runs; `poison` +// adds a fixed value to one expected element, which the bar must see +def private linear_case_kq(nin, nout : int64; decode, repack : bool; poison : bool = false) : BlockCase { + var r = TtsRng() + rng_seed(r, 0xebcul + uint64(nin)) + let t = decode ? 1l : T_ROWS + var l = TtsLinear(nin = nin, nout = nout) + rand_fill(r, l.b, nout, 0.5) + var kq : array + var ks : array + build_q4k_planes(kq, ks, nout, nin, 0x51ED270Bu) + var inscope x : array + rand_fill(r, x, t * nin, 2.0) + var inscope want : array + kq_leaf_rows(kq, ks, x, t, nin, nout, l.b, want) + if (poison) { + want[length(want) / 2] += 4.0 // a fixed value: the outputs run to a few thousand, so a quarter would hide inside the relative bar + } + linear_take_kq(l, 4, kq, ks, repack) // the planes move into the layer + var inscope got : array + if (decode) { + linear_rows_decode(l, x, got) + } else { + linear_rows(l, x, t, got) + } + var inscope env : array + env |> resize(t * nout) + for (i in range64(t * nout)) { + env[i] = abs(want[i]) + 1.0 + } + let name = "linear kq {nin}x{nout} {decode ? "decode" : "rows"}{repack ? "" : " disk-order"}{poison ? " poisoned" : ""}" + var res = score_at(name, got, want, env, 0.0, KQ_REL) + if (l.kq_fmt != 4 || !empty(l.w) || (repack && kernel_backend_has_kq() != l.kq_repacked)) { + res.bad += 1000000 // the kq lane never engaged, or the repack did not follow the backend + } + return res +} + +// a width the 256-superblock does not divide is refused by name: a case with one bad element +// when the panic text is not the layer's, none when it is +def private kq_width_refusal() : BlockCase { + var res = BlockCase(name = "linear kq width refusal") + var why = "no panic" + try { + var l = TtsLinear(nin = 100l, nout = 8l) + var kq : array + var ks : array + linear_take_kq(l, 4, kq, ks, false) + } recover { + why = this_context().last_exception |> rtrim + } + if (why |> find("needs nin on 256") < 0) { + res.bad = 1 + res.name = "linear kq width refusal - got '{why}'" + } + return res +} + // 66 input channels padded to 96 quantize per 32, so the padded conv reaches the q8 lane: the // padded rows form against the unpadded channel-major reference at the q8 bar. def private pad_case_q8() : BlockCase { @@ -926,6 +1060,15 @@ def test_rows_convs(t : T?) { report(t, conv_case_q8("conv q8 k11 d5", 256l, 256l, 11l, 25l, 5l)) report(t, linear_case_q8(768l, 768l)) } + t |> run("the K-quant lane against the leaf per row - both arms, and a width off the float4 bias") @(t : T?) { + report(t, linear_case_kq(512l, 96l, false, true)) + report(t, linear_case_kq(768l, 64l, true, true)) + report(t, linear_case_kq(512l, 96l, false, false)) + report(t, linear_case_kq(768l, 64l, true, false)) + report(t, linear_case_kq(256l, 6l, true, true)) // nout off 4: the bias tail past the last float4 + report_poison(t, linear_case_kq(512l, 96l, false, true, true)) + report(t, kq_width_refusal()) + } t |> run("snake, adain, pad, polar") @(t : T?) { report(t, snake_case()) report(t, adain_case(64l)) diff --git a/modules/dasLLAMA/tests/test_tts_pocket.das b/modules/dasLLAMA/tests/test_tts_pocket.das index 421f068b78..98dcfa73de 100644 --- a/modules/dasLLAMA/tests/test_tts_pocket.das +++ b/modules/dasLLAMA/tests/test_tts_pocket.das @@ -623,6 +623,135 @@ def test_pocket_q8_file(t : T?) { } let Q8_FILE_BAR = 5.0e-2lf +let KQ_EXACT_BAR = 5.0e-2lf //! the kq lane against the exact lane of the same file (measured 3.4e-2): the head's Q8_0 route and the activation quants +let KQ_HEAD_BAR = 2.0e-2lf //! the flow head on its Q8_0 route against the f32 oracle's frames (measured 9.8e-3), the oracle's own conditioning and noise in + +def private kq_gguf_path() : string { + return path_join(models_dir(), "pocket-tts-en-kq.gguf") +} + +[test] +def test_pocket_kq_file(t : T?) { + //! The small form (`convert_pocket.py --kq`): the backbone's and the codec transformers' + //! matrices and the text embedding as Q4_K, the flow head as Q8_0. Unpinned, the K-quant + //! tensors serve as their own planes (the kq lane); pinned q8 or f32 they dequantize into + //! that lane. The three lanes of one file against each other and against the f16 reference. + let path = gguf_path() + let kqpath = kq_gguf_path() + if (!model_available(t, path) || !model_available(t, kqpath)) { + return + } + var inscope man <- load_manifest() + if (empty(man.cases)) { + t |> skip("no oracle dumps under {oracle_dir()}") + return + } + with_job_que() { + setup_dasllama_jobque_() + var inscope sc = PocketScratch() + defer() { + reset_pocket_q8() + } + reset_pocket_q8() + t |> success(pocket_serves_native(), "unpinned, a K-quant tensor serves as its own planes") + var inscope mk <- load_pocket(kqpath) + var kq_layers = 0 + for (l in mk.backbone.layers) { + kq_layers += (l.in_proj.kq_fmt == 4 && l.out_proj.kq_fmt == 4 && l.ffn1.kq_fmt == 4 && l.ffn2.kq_fmt == 4) ? 1 : 0 + } + t |> equal(kq_layers, length(mk.backbone.layers), "every backbone layer's four GEMMs came in as Q4_K planes") + t |> success(mk.mimi.dec_tf.layers[0].ffn1.kq_fmt == 4 && mk.mimi.enc_tf.layers[0].in_proj.kq_fmt == 4, "the codec transformers' matrices too") + t |> success(mk.head.blocks[0].mlp1.q8 && mk.head.final_linear.q8 && mk.head.cond_embed.q8 && mk.head.input_proj.q8, "the flow head's matrices came in as Q8_0 blocks, its 32-wide input projection included") + t |> success(!mk.speaker_proj.q8 && mk.speaker_proj.kq_fmt == 0, "the speaker projection stays f32") + t |> equal(long_length(mk.text_emb.a), 4001l * mk.backbone.d, "the Q4_K embedding table dequantized into its f32 form") + // the same file on the q8 lane: every Q4_K tensor dequantized and requantized to Q8_0 (a + // re-encode of the same values), so the two lanes differ by that re-encode and their + // activation quants; the head is the same Q8_0 blocks on both + set_pocket_q8(true) + var inscope mq <- load_pocket(kqpath) + t |> success(mq.backbone.layers[0].in_proj.q8 && mq.backbone.layers[0].in_proj.kq_fmt == 0, "pinned q8, the K-quant tensor is Q8_0 blocks") + // the exact lane over the K-quant file: dequantized f32 planes, the head dequantized too - + // the one load whose head is not the Q8_0 route, so it is what pins that route + set_pocket_q8(false) + var inscope m32 <- load_pocket(kqpath) + t |> success(!m32.q8 && m32.backbone.layers[0].in_proj.kq_fmt == 0 && !m32.backbone.layers[0].in_proj.q8, "pinned f32, the K-quant tensor is an f32 plane") + t |> success(!m32.head.blocks[0].mlp1.q8, "pinned f32, the head is f32 too") + // the f16 file on its f32 lane: the reference the whole small form is measured against + var inscope mf <- load_pocket(path) + t |> success(!mf.q8 && !mf.backbone.layers[0].in_proj.q8, "the f16 reference serves its f32 lane") + reset_pocket_q8() + var compared = 0 + for (c in man.cases) { + continue if (!c.stages || c.voice != "alba") + compared++ + kq_lanes_case(t, c, man, mk, mq, m32, mf, sc) + break + } + t |> success(compared > 0, "the oracle manifest carries a stages case in alba - the lanes were compared") + kq_clone_over_roster(t, kqpath) + } +} + +// the three lanes of the small file on one oracle case: kq against q8 at the q8 bar (with the +// poisoned control), kq against the exact lane (the head's Q8_0 route), the f16 reference logged; +// then the stored roster against the reference encoder (the f16 file on its f32 lane) +def private kq_lanes_case(t : T?; c : PocketCase; man : PocketManifest; var mk, mq, m32, mf : PocketModel; var sc : PocketScratch) { + var inscope a <- forced_latents(mk, c, man, sc) + var inscope b <- forced_latents(mq, c, man, sc) + let rel = rms_relative(a, b) + to_log(LOG_INFO, "pocket {c.id} kq lane vs the q8 lane of the same file: rms relative {rel}\n") + t |> success(rel < Q8_FILE_BAR, "{c.id}: the kq lane and the q8 lane of one file agree (rms relative {rel})") + var inscope poisoned := b + for (v in poisoned) { + v += 2.0 + } + t |> success(rms_relative(a, poisoned) >= Q8_FILE_BAR, "{c.id}: the bar discriminates a poisoned expectation") + var inscope e <- forced_latents(m32, c, man, sc) + let rel32 = rms_relative(a, e) + to_log(LOG_INFO, "pocket {c.id} kq lane vs the exact lane of the same file: rms relative {rel32} (the head's Q8_0 route and the activation quants)\n") + t |> success(rel32 < KQ_EXACT_BAR, "{c.id}: the kq lane and the exact lane of one file agree (rms relative {rel32})") + var inscope f <- forced_latents(mf, c, man, sc) + to_log(LOG_INFO, "pocket {c.id} kq lane vs the f16 reference: rms relative {rms_relative(a, f)} (the format's loss; the rig is its gate)\n") + var inscope dump <- load_tts_dump(path_join(oracle_dir(), c.file)) + // the head alone on its Q8_0 route - Q8_0 blocks and a Q8_0-quantized activation vector per + // GEMV - fed the oracle's own conditioning and noise, against the oracle's frames + var inscope x0 := dump["x0"].f + var inscope hk <- head_over_frames(mk, sc, dump["cond"].f, x0, int64(c.n_frames)) + let rel_head = rms_relative(hk, dump["x1"].f) + to_log(LOG_INFO, "pocket {c.id} the flow head on Q8_0 blocks vs the oracle: rms relative {rel_head}\n") + t |> success(rel_head < KQ_HEAD_BAR, "{c.id}: the head's Q8_0 route holds the oracle's frames (rms relative {rel_head})") + var inscope head_poisoned := dump["x1"].f + for (v in head_poisoned) { + v += 2.0 + } + t |> success(rms_relative(hk, head_poisoned) >= KQ_HEAD_BAR, "{c.id}: the head bar discriminates a poisoned expectation") + var inscope lat : array + let frames = pocket_encode_latents(mf, dump["voice_pcm"].f, sc, lat) // the f16 file's encoder, f32 weights: the parity rail's own; the kq file's carries Q4_K and Q8_0 + var inscope stored : array + mk.voice_latents |> get("alba") $(s) { + stored := s + } + t |> equal(long_length(stored), frames * mk.latent_dim, "alba's stored frames are the clip's frame count") + rel_diff(t, "{c.id} stored roster vs the reference encoder", stored, lat, ENCODER_BAR) +} + +// a clone over a roster name: alba re-registered from a clip keeps the roster at 19 and speaks +def private kq_clone_over_roster(t : T?; kqpath : string) { + var inscope mkf <- load_tts_model(kqpath) + var inscope clip : array + clip |> resize(24000 * 3) + var r = TtsRng() + rng_seed(r, 7ul) + for (i in range(length(clip))) { + clip[i] = 0.3 * sin(float(i) * 0.05) * (0.5 + 0.5 * rng_normal(r)) + } + tts_register_voice(mkf, "alba", clip, 24000) + var inscope ck <- caps(mkf) + t |> equal(length(ck.voices), 19, "a clone under a roster name replaces the voice, the roster count stays") + var inscope spoken <- synthesize(mkf, "The quick brown fox jumps over the lazy dog.", "alba") + let st = speech_stats(spoken.pcm) + t |> success(st.finite && st.rms > 0.01, "the replaced roster voice speaks (rms {st.rms})") +} // ===== the other five languages: one Q8_0 file each, its own tokenizer, one default voice ===== @@ -781,6 +910,7 @@ def test_pocket_facade(t : T?) { m.pocket.voices |> get("caro_davy") $(pcm) { clip := pcm } + t |> success(!empty(clip), "the parity file stores its roster as clips (the older form) - caro_davy's is the clone's source") tts_register_voice(m, "cloned", clip, 24000) var inscope c2 <- caps(m) t |> equal(length(c2.voices), 20, "the cloned voice joined the roster") @@ -828,6 +958,41 @@ def private pocket_refusals(t : T?; var m : TtsModel; clip : array) { t |> equal(find_index(c3.voices, "endless"), -1, "and left no half voice in the roster") } +// the file shape a demo ships: one voice stored as its latent frames, the codec encoder left out +// (`convert_pocket.py --voices stuart_bell --no-cloning --kq`) - it speaks its roster from the +// stored frames, says it cannot clone, and refuses a clip by name +[test] +def test_pocket_no_encoder(t : T?) { + let path = path_join(models_dir(), "pocket-tts-en-stuart-kq.gguf") + if (!model_available(t, path)) { + return + } + with_job_que() { + setup_dasllama_jobque_() + t |> success(!tts_needs_packs(path), "a Pocket file needs no packs") + var inscope m <- load_tts_model(path) + var inscope c <- caps(m) + t |> success(!c.cloning, "a file without the encoder does not clone") + t |> success(!c.speed, "and has no speed to honour") + t |> equal(length(c.voices), 1, "one voice in the roster") + t |> equal(c.voices[0], "stuart_bell") + t |> equal(length(c.langs), 1) + t |> equal(c.langs[0], "en") + var inscope a <- synthesize(m, "Whose woods these are I think I know.", "stuart_bell") + let st = speech_stats(a.pcm) + t |> success(st.finite && st.rms > 0.01, "the stored voice speaks (rms {st.rms})") + t |> success(length(a.pcm) >= 24000 && length(a.pcm) <= 24000 * 8, "speech length {length(a.pcm)} samples") + var inscope clip : array + clip |> resize(24000) + let why = panic_text_of() { + tts_register_voice(m, "x", clip, 24000) + } + t |> success(why |> find("no codec encoder") >= 0, "a clip is refused by the missing encoder: {why}") + var inscope c2 <- caps(m) + t |> equal(length(c2.voices), 1, "and the roster is unchanged") + } +} + // a text the chunker cannot split - no sentence mark, no comma - is one chunk past the token // budget: the voice's caches grow to hold its text and every frame the cap allows, and the // synthesis runs to EOS instead of panicking on the cache diff --git a/site-dasllama/REVIEW.md b/site-dasllama/REVIEW.md index 9897a7e160..739dd0e1e1 100644 --- a/site-dasllama/REVIEW.md +++ b/site-dasllama/REVIEW.md @@ -46,7 +46,8 @@ publish time is a defect.** A dated `_news` or `_stories` entry is read as a cla date; standing page copy - a masthead, section prose, a meta tag - is read as a claim about now. `README.md`'s copy rules say how a claim is checked. -**A diff that falsifies standing page copy - a masthead, section prose, a meta tag - fixes it +**A diff that falsifies standing page copy - a masthead, section prose, a meta tag - or a +`_news/*.md` / `_stories/*.md` entry dated on or after the publish day of this change, fixes it in the same change.** **A PR whose copy describes what a linked download contains refreshes that artifact at its diff --git a/site-dasllama/_news/2026-09-10-parrot.md b/site-dasllama/_news/2026-09-10-parrot.md new file mode 100644 index 0000000000..ec29f5afc5 --- /dev/null +++ b/site-dasllama/_news/2026-09-10-parrot.md @@ -0,0 +1,14 @@ +--- +date: 2026-09-10 +tag: examples +title: Parrot - talk for a few seconds, and the browser reads the poem in your voice. +--- + +A third dasLLAMA example on the examples page. Press record and talk; Silero VAD ends the take +when you go quiet, Pocket TTS clones the voice from it, and the text in the box - Frost's +"Stopping by Woods on a Snowy Evening" to begin with, or whatever you type - is read in that +voice. The recording stays in the tab. Behind it the English Pocket file shrank from 152 MB to +75: the backbone and the codec transformers as Q4_K, the flow head Q8_0, the codec encoder +still inside; on our 200-sentence rig it reads 3.86 WER / 4.295 UTMOS against the q8 file's +3.91 / 4.328, and we could not hear the difference. Storywish now reads through the same form +with one voice and no encoder, 65 MB. [Try it](examples.html). diff --git a/site-dasllama/_news/2026-09-10-storywish.md b/site-dasllama/_news/2026-09-10-storywish.md index 57b884e054..4d1872ea2d 100644 --- a/site-dasllama/_news/2026-09-10-storywish.md +++ b/site-dasllama/_news/2026-09-10-storywish.md @@ -9,8 +9,8 @@ GPT-Neo, which no GGUF engine runs, so we trained our own: `tinystories-instruct 27M-parameter llama on the TinyStoriesInstruct corpus with a 4K vocabulary, 15 minutes on one H100. Asked for three words over five word triples, 24 sampled stories each, it puts all three in 61 of the 120 stories; the official 33M puts them in 60. It -ships as a 32 MB `.dlim` beside KittenTTS nano; you type the words, Enter tells the story, Tab -asks for dialogue. The model, the training recipe and the hit-rate numbers are on +ships as a 32 MB `.dlim` beside a 65 MB Pocket TTS file with one voice inside and no phoneme +packs; you type the words, Enter tells the story, Tab asks for dialogue. The model, the training recipe and the hit-rate numbers are on [Hugging Face](https://huggingface.co/borisbat/dasllama-stories). The model sets of both examples are now minted by the deploy for the build it ships, so a format bump can no longer leave a page silently declining its images. [Try it](examples.html). diff --git a/site-dasllama/examples.html b/site-dasllama/examples.html index 34fb63282f..fb1fff5255 100644 --- a/site-dasllama/examples.html +++ b/site-dasllama/examples.html @@ -66,12 +66,28 @@

Storyteller

Storywish

llm + tts · in the browser -

Type the words you wish for and a story model trained to take requests - our own 27M llama on the TinyStoriesInstruct corpus, published on Hugging Face as borisbat/dasllama-stories - writes a children's tale that uses them while KittenTTS reads it aloud. The same wasm64 engine as the storyteller, a 32 MB model image.

+

Type the words you wish for and a story model trained to take requests - our own 27M llama on the TinyStoriesInstruct corpus, published on Hugging Face as borisbat/dasllama-stories - writes a children's tale that uses them while Pocket TTS reads it aloud in one baked voice, no phoneme front end in the loop. The same wasm64 engine as the storyteller, a 32 MB story image and the speech model's own file.

llmttswasm64threads
type · Enter tells the story · Tab asks for dialogueChrome or Edge 133+, Firefox 134+ (memory64)
+ +
+ Parrot poster + +
+
+
+

Parrot

+ voice cloning · in the browser +
+

Press record and talk for a few seconds. Pocket TTS clones your voice from the take and reads the text in the box - a poem to begin with, or whatever you type - in it; Silero VAD ends the take when you go quiet, and you can record again at any time. The recording stays in the tab: nothing is uploaded. The same wasm64 engine, the speech model's 75 MB file with its codec encoder, and a 1 MB voice-activity model.

+
ttsvoice cloningwasm64threads
+
record · talk · sayChrome or Edge 133+, Firefox 134+ (memory64)
+
+ +

Source for every example lives under examples/dasLLAMA in the repository; the browser builds are made with daspkg release wasm from the same sources.

diff --git a/site-dasllama/feed.xml b/site-dasllama/feed.xml index 44cfdf7c38..3bf5f86a6c 100644 --- a/site-dasllama/feed.xml +++ b/site-dasllama/feed.xml @@ -16,13 +16,27 @@ GPT-Neo, which no GGUF engine runs, so we trained our own: <code>tinystori 27M-parameter llama on the TinyStoriesInstruct corpus with a 4K vocabulary, 15 minutes on one H100. Asked for three words over five word triples, 24 sampled stories each, it puts all three in 61 of the 120 stories; the official 33M puts them in 60. It -ships as a 32 MB <code>.dlim</code> beside KittenTTS nano; you type the words, Enter tells the story, Tab -asks for dialogue. The model, the training recipe and the hit-rate numbers are on +ships as a 32 MB <code>.dlim</code> beside a 65 MB Pocket TTS file with one voice inside and no phoneme +packs; you type the words, Enter tells the story, Tab asks for dialogue. The model, the training recipe and the hit-rate numbers are on <a href="https://huggingface.co/borisbat/dasllama-stories">Hugging Face</a>. The model sets of both examples are now minted by the deploy for the build it ships, so a format bump can no longer leave a page silently declining its images. <a href="examples.html">Try it</a>.</p> +Parrot - talk for a few seconds, and the browser reads the poem in your voice. + +https://dasllama.io/#n-2026-09-10-parrot +2026-09-10T00:00:00Z +<p>A third dasLLAMA example on the examples page. Press record and talk; Silero VAD ends the take +when you go quiet, Pocket TTS clones the voice from it, and the text in the box - Frost's +"Stopping by Woods on a Snowy Evening" to begin with, or whatever you type - is read in that +voice. The recording stays in the tab. Behind it the English Pocket file shrank from 152 MB to +75: the backbone and the codec transformers as Q4_K, the flow head Q8_0, the codec encoder +still inside; on our 200-sentence rig it reads 3.86 WER / 4.295 UTMOS against the q8 file's +3.91 / 4.328, and we could not hear the difference. Storywish now reads through the same form +with one voice and no encoder, 65 MB. <a href="examples.html">Try it</a>.</p> + + dasllama-server is a download now - one archive per platform. https://dasllama.io/#n-2026-09-07-download-dasllama-server diff --git a/site-dasllama/files/examples/parrot-poster.jpg b/site-dasllama/files/examples/parrot-poster.jpg new file mode 100644 index 0000000000..5626cf4459 Binary files /dev/null and b/site-dasllama/files/examples/parrot-poster.jpg differ diff --git a/site-dasllama/index.html b/site-dasllama/index.html index 16ba6a68fe..798efd4097 100644 --- a/site-dasllama/index.html +++ b/site-dasllama/index.html @@ -70,12 +70,23 @@

Local inference, written in daslang.

27M-parameter llama on the TinyStoriesInstruct corpus with a 4K vocabulary, 15 minutes on one H100. Asked for three words over five word triples, 24 sampled stories each, it puts all three in 61 of the 120 stories; the official 33M puts them in 60. It -ships as a 32 MB .dlim beside KittenTTS nano; you type the words, Enter tells the story, Tab -asks for dialogue. The model, the training recipe and the hit-rate numbers are on +ships as a 32 MB .dlim beside a 65 MB Pocket TTS file with one voice inside and no phoneme +packs; you type the words, Enter tells the story, Tab asks for dialogue. The model, the training recipe and the hit-rate numbers are on Hugging Face. The model sets of both examples are now minted by the deploy for the build it ships, so a format bump can no longer leave a page silently declining its images. Try it.

+
+
2026-09-10examples

Parrot - talk for a few seconds, and the browser reads the poem in your voice.

▾
+

A third dasLLAMA example on the examples page. Press record and talk; Silero VAD ends the take +when you go quiet, Pocket TTS clones the voice from it, and the text in the box - Frost's +"Stopping by Woods on a Snowy Evening" to begin with, or whatever you type - is read in that +voice. The recording stays in the tab. Behind it the English Pocket file shrank from 152 MB to +75: the backbone and the codec transformers as Q4_K, the flow head Q8_0, the codec encoder +still inside; on our 200-sentence rig it reads 3.86 WER / 4.295 UTMOS against the q8 file's +3.91 / 4.328, and we could not hear the difference. Storywish now reads through the same form +with one voice and no encoder, 65 MB. Try it.

+
2026-09-07site

dasllama-server is a download now - one archive per platform.

▾

The OpenAI-compatible server ships as a standalone bundle for macOS (Apple silicon), Windows diff --git a/site-dasllama/test_metadata.py b/site-dasllama/test_metadata.py index 8602c1cdbd..00d7a40e96 100644 --- a/site-dasllama/test_metadata.py +++ b/site-dasllama/test_metadata.py @@ -110,6 +110,7 @@ def test_caddy_redirects_explicit_index(self): EXAMPLE_SHELLS = { "storyteller": (REPO_ROOT / "examples" / "dasLLAMA" / "storyteller" / "web_shell.html", "runStoryteller"), "storywish": (REPO_ROOT / "examples" / "dasLLAMA" / "storywish" / "web_shell.html", "runStorywish"), + "parrot": (REPO_ROOT / "examples" / "dasLLAMA" / "parrot" / "web_shell.html", "runParrot"), } diff --git a/utils/dasllama-server/README.md b/utils/dasllama-server/README.md index 8af526e6ad..3913449d04 100644 --- a/utils/dasllama-server/README.md +++ b/utils/dasllama-server/README.md @@ -184,10 +184,11 @@ Catalog entries carry their **towers**: a vision-capable row offers its pinned m under the table offers the ASR tower (parakeet v3; wires the `asr` key the same way) - `POST /catalog/download` takes `{"name", "tower": "vision"}` or `{"tower": "asr"}` on the same one-at-a-time rail. A **speech** strip sits beside it for the text-to-speech set the -`/catalog` document's `tts` list carries: the two front-end packs first (every speech model -loads them), then one model, then **enable speech** wires the `tts` key - -`{"tower": "tts", "file": }` pulls one file of that set on the same rail; a Pocket file is -its whole set. Setup-mode +`/catalog` document's `tts` list carries: a model on disk that can speak goes straight to +**enable speech**, which wires the `tts` key (a Pocket file stands alone; a phoneme family +needs the two front-end packs beside it - `needs_packs` on the row); otherwise the packs first, +then one model - `{"tower": "tts", "file": }` pulls one file of that set on the same rail. +Setup-mode **serve this model** wires any tower already on disk automatically. Each row also wears a **fit badge** (fits gpu / fits / tight / too big) from the box facts the `/catalog` document carries (`box.ram_gb`, and the armed tier's weight @@ -309,7 +310,7 @@ server first; Windows locks the DLLs. | `POST` | `/v1/audio/speech` | Text->speech (needs `--tts`): `{"input", "voice"?, "speed"?, "response_format"?: "wav" \| "pcm"}` - the OpenAI shape; `wav` (default) is 16-bit PCM at the model's rate, `pcm` the raw samples; the compressed formats answer `400` (no encoder here). One synthesis at a time on the TTS worker (its kernels run inline under `hybrid`, like the ASR workers'), 16 queued | | `POST` | `/v1/audio/phonemes` | The front end alone (needs `--tts`): `{"model"?, "input", "voice"?}` -> `{"normalized", "lang", "chunks": [{"text", "phonemes"}]}` - the normalizer's spoken form of the text, the dialect the voice speaks (`lang`), then one row per chunk a synthesis of it would take, each carrying that chunk beside its phoneme string in that dialect. A model whose front end phonemizes ONE language reads every voice name in it - an alias, or a name it does not carry, since there is no other answer to give; a model that phonemizes several requires a voice from its `caps` and refuses an unservable one with the speech route's own 400. `model` is read the way the speech route reads it (`404` on an id that is not the served one). Answered by the TTS worker on the same queue as a synthesis (the same 4096-CHARACTER cap - codepoints, not bytes - and the same 503 when no speech model is served), so the speech studio can show what the model will actually say | | `POST` | `/vad` | Silero speech spans over an uploaded clip (the control page's waveform overlay; in-handler, <=120 s, needs the in-repo `silero_vad.bin`) | -| `GET` | `/catalog` | The curated model list with local presence, the `asr` tower row, the `tts` list (the three speech GGUFs and the two front-end packs, each `file`/`bytes`/`pack`/`present`/`path`), the `box` memory facts + the download state machine (`idle | downloading | verifying | done | failed`, byte progress) | +| `GET` | `/catalog` | The curated model list with local presence, the `asr` tower row, the `tts` list (the two front-end packs the speech route loads, then every served speech GGUF, each `file`/`bytes`/`pack`/`present`/`path`/`needs_packs` - on a model, whether the file on disk reads the packs, true until it is here; false on a pack), the `box` memory facts + the download state machine (`idle | downloading | verifying | done | failed`, byte progress) | | `POST` | `/catalog/download` | `{"name": }` - start one catalog download; `{"name", "tower": "vision"}` / `{"tower": "asr"}` pull a tower, `{"tower": "tts", "file": }` one file of the speech set (409 while one runs or the file exists; sha-verified, never waived) | | `POST` | `/bench` | Loopback-only: start the benchmark, quiesced. In process by default: pp512 and tg128 on the served model, an untimed warmup then three timed reps each, one pp prefill or one tg token per tick, the text, audio and speech routes (503) and the model-switching, bake and catalog-download routes (409) holding until it finishes - in either mode; with `lcpp_bin` in the config on a source-tree daslang, the A/B child instead - our lcpp_bench then llama-bench on the same GGUF. 400 in setup mode or when the served context is shorter than pp512, 409 while a bench, a bake or a catalog download runs or streams are active, 503 while draining | | `GET` | `/bench` | Bench state (`idle | running | done | failed`), `mode` (`inprocess` | `ab`: what a `POST` runs), live log lines, the result JSON - `ours_pp`, `ours_tg`, `threads`, `elapsed_s`, `ts`; in process also `mode`, `pp_sd`, `tg_sd`, `reps`, `model`, `gguf`, `backend` (`metal`, `gpu:resident`, or the slot's word), `kv` (the codec the rows ran), `exec` (`exe-native` | `jit` | `interpreted`), `tune` (`fat` | `tuned` | `untuned (N of M on fallback)` | `none`) and `ref_cmd` (the llama-bench line for the comparison); the A/B also `theirs_pp`, `theirs_tg`, `pp_ratio`, `tg_ratio` and `record` - and the hardware line | diff --git a/utils/dasllama-server/control.html b/utils/dasllama-server/control.html index aeb6da1c3a..6abd3059af 100644 --- a/utils/dasllama-server/control.html +++ b/utils/dasllama-server/control.html @@ -3238,10 +3238,12 @@ strip.append(b, label); } -// the download ladder over /catalog's `tts` list: the two front-end packs first (every speech -// model loads them), then the models, then the enable. The rail runs one download at a time, -// so an absent pack pair is one button and the card re-offers what is still missing. -// False back = the catalog has no tts list to offer from, and the caller says so instead. +// the download ladder over /catalog's `tts` list: a model on disk that can speak - a Pocket file +// stands alone, a phoneme family reads the two front-end packs beside it - goes straight to the +// enable; otherwise the packs first (every phoneme family loads them), then the models. The rail +// runs one download at a time, so an absent pack pair is one button and the card re-offers what +// is still missing. False back = the catalog has no tts list to offer from, and the caller says +// so instead. function ttsOfferArm(strip, dl, kept) { const items = (catState && catState.tts) || []; if (!items.length) return false; @@ -3253,30 +3255,31 @@ return true; } const missingPacks = items.filter(i => i.pack && !i.present); + const models = items.filter(i => !i.pack); + const present = models.filter(m => m.present && (!m.needs_packs || !missingPacks.length)) + .sort((a, b) => a.bytes - b.bytes); + if (present.length) { + ttsEnableArm(strip, label, present, kept); + return true; + } if (missingPacks.length) { const bytes = missingPacks.reduce((a, p) => a + p.bytes, 0); const b = miniButton("download the front-end packs (" + fmtSize(bytes) + ")", () => startTtsDownload(missingPacks[0].file)); - b.title = "the text normalizer and grapheme-to-phoneme tables every speech model loads"; + b.title = "the text normalizer and grapheme-to-phoneme tables every phoneme family loads"; b.disabled = dlBusy(dl); label.textContent = "add speech synthesis — /v1/audio/speech and the speech studio above"; strip.append(b, label); return true; } - const models = items.filter(i => !i.pack); - const present = models.filter(m => m.present).sort((a, b) => a.bytes - b.bytes); - if (!present.length) { - label.textContent = "the front-end packs are here — pick a speech model"; - for (const m of models) { - const b = miniButton("download " + ttsName(m.file) + " (" + fmtSize(m.bytes) + ")", - () => startTtsDownload(m.file)); - b.disabled = dlBusy(dl); - strip.append(b); - } - strip.append(label); - return true; + label.textContent = "the front-end packs are here — pick a speech model"; + for (const m of models) { + const b = miniButton("download " + ttsName(m.file) + " (" + fmtSize(m.bytes) + ")", + () => startTtsDownload(m.file)); + b.disabled = dlBusy(dl); + strip.append(b); } - ttsEnableArm(strip, label, present, kept); + strip.append(label); return true; } diff --git a/utils/dasllama-server/model_catalog.das b/utils/dasllama-server/model_catalog.das index d4a5fcbebf..70164d94f3 100644 --- a/utils/dasllama-server/model_catalog.das +++ b/utils/dasllama-server/model_catalog.das @@ -16,6 +16,12 @@ require daslib/jobque_boost require daslib/strings_boost require strings +//! The front-end packs the speech route loads beside a phoneme family's GGUF - the pack rows the +//! /catalog document's `tts` list carries. A companion of the served set outside this list (the +//! browser's American-only phoneme pack) is fetchable through `catalog_download_start_tts` and +//! rides no ladder. +let public TTS_FRONT_END_PACKS : array <- ["tts_g2p.bin", "tts_postag.bin"] + //! One curated model: a commit-pinned, ungated HF file this engine serves today. //! `vram_hint_gb` is the advertised working-set hint, not a fit check. The `vision_*` //! fields carry the row's pinned vision tower (mmproj) when it has one — "" = text only. @@ -341,9 +347,11 @@ def public catalog_download_tick() { } //! The GET /catalog document: entries with local presence (model AND vision tower), the ASR -//! tower, the box's memory facts (`ram_gb` in GiB, `vram_mb` in MiB, 0 = unknown — the page's fit badges), -//! plus the download state machine. -def public catalog_state_json(dir : string; ram_gb : int = 0; vram_mb : int64 = 0l) : string { +//! tower, the `tts` list (the two front-end packs, then every served speech model; a model row's +//! `needs_packs` is `needs_packs(path)` on the file once it is here - the server passes the +//! engine's family test - and true until then), the box's memory facts (`ram_gb` in GiB, +//! `vram_mb` in MiB, 0 = unknown — the page's fit badges), plus the download state machine. +def public catalog_state_json(dir : string; ram_gb : int; vram_mb : int64; needs_packs : function<(path : string) : bool>) : string { var cat <- model_catalog() var entries : array entries |> reserve(length(cat)) @@ -367,10 +375,12 @@ def public catalog_state_json(dir : string; ram_gb : int = 0; vram_mb : int64 = var tts_items : array tts_items |> reserve(length(tts_set)) for (item in tts_set) { + continue if (item.pack && !has_value(TTS_FRONT_END_PACKS, item.entry.name)) let p = path_join(dir, item.entry.name) let present = stat(p).is_valid + let packs = !item.pack && (!present || invoke(needs_packs, p)) tts_items |> push(JV((file = item.entry.name, bytes = item.entry.bytes, pack = item.pack, - present = present, path = present ? p : ""))) + present = present, path = present ? p : "", needs_packs = packs))) } var got = 0l if (g_dl_state == "downloading") { diff --git a/utils/dasllama-server/openai_server.das b/utils/dasllama-server/openai_server.das index e36d99725e..413dacad7e 100644 --- a/utils/dasllama-server/openai_server.das +++ b/utils/dasllama-server/openai_server.das @@ -179,7 +179,6 @@ let TTS_LANE_DEFAULT = "q8" var g_tts_lane = TTS_LANE_DEFAULT // the weight lane the worker is ASKED for; its ready event echoes the pin it took var g_tts_voices_dir = "" // clips a cloning model adds to its voices at boot, each under its file's stem let TTS_VOICE_CLIP_EXTENSIONS : array <- [".wav", ".flac", ".mp3", ".ogg"] // what the decode rail reads -let TTS_FRONT_END_PACKS : array <- ["tts_g2p.bin", "tts_postag.bin"] // load_tts_model reads both from the GGUF's directory var g_shutdown_requested = false var g_gc_requested = false var g_req_counter = 0l @@ -5285,7 +5284,7 @@ class OpenAiServer : HvWebServer { return handle_bake_start(req, resp) } GET("/catalog") <| @(var req : HttpRequest?; var resp : HttpResponse?) : http_status { - return resp |> JSON(catalog_state_json(g_catalog_dir, g_box_ram_gb, gpu_weight_budget_bytes() / (1024l * 1024l))) + return resp |> JSON(catalog_state_json(g_catalog_dir, g_box_ram_gb, gpu_weight_budget_bytes() / (1024l * 1024l), @@tts_needs_packs)) } POST("/catalog/download") <| @(var req : HttpRequest?; var resp : HttpResponse?) : http_status { if (!is_loopback_req(req)) return deny_operator(resp) diff --git a/utils/dasllama-server/test_model_catalog.das b/utils/dasllama-server/test_model_catalog.das index 604a34dca1..ffdbb25012 100644 --- a/utils/dasllama-server/test_model_catalog.das +++ b/utils/dasllama-server/test_model_catalog.das @@ -92,10 +92,10 @@ def test_catalog_table(t : T?) { packs++ } } - t |> equal(length(set), 12, "the set is nine models (three phoneme families, six Pocket languages) and three packs") + t |> equal(length(set), 14, "the set is eleven models (three phoneme families, six Pocket languages, the two small English Pocket forms) and three packs") t |> equal(packs, 3, "three front-end packs: the full phoneme pack, its American-only twin, the tagger") for (want in ["kitten-nano.gguf", "kitten-mini.gguf", "kokoro-82m.gguf", "tts_g2p.bin", "tts_g2p_en_us.bin", "tts_postag.bin", - "pocket-tts-en-q8.gguf", "pocket-tts-de-q8.gguf", "pocket-tts-es-q8.gguf", "pocket-tts-it-q8.gguf", "pocket-tts-pt-q8.gguf", "pocket-tts-fr-q8.gguf"]) { + "pocket-tts-en-q8.gguf", "pocket-tts-de-q8.gguf", "pocket-tts-es-q8.gguf", "pocket-tts-it-q8.gguf", "pocket-tts-pt-q8.gguf", "pocket-tts-fr-q8.gguf", "pocket-tts-en-kq.gguf", "pocket-tts-en-stuart-kq.gguf"]) { t |> success(key_exists(names, want), "the set carries {want}") } } @@ -183,12 +183,18 @@ def test_download_refusals(t : T?) { } } +// the family test the document is handed in place of the engine's: a file named as a Pocket +// form reads text and needs no pack, everything else is a phoneme family +def private needs_packs_by_name(path : string) : bool { + return find(base_name(path), "pocket-tts-") < 0 +} + [test] def test_state_json(t : T?) { t |> run("the /catalog document carries every entry with presence and an idle machine") @(t : T?) { let dir = tmp_models_dir("doc") var jerr = "" - var doc = read_json(catalog_state_json(dir, 64, 8192l), jerr) + var doc = read_json(catalog_state_json(dir, 64, 8192l, @@needs_packs_by_name), jerr) t |> success(doc != null, "state json parses: {jerr}") if (doc != null) { t |> equal(doc?["models_dir"] ?? "", dir) @@ -228,7 +234,7 @@ def test_state_json(t : T?) { } } var jerr = "" - var doc = read_json(catalog_state_json(dir), jerr) + var doc = read_json(catalog_state_json(dir, 0, 0l, @@needs_packs_by_name), jerr) t |> success(doc != null, "state json parses: {jerr}") let entries = doc?["entries"] if (entries != null && (entries.value is _array)) { @@ -250,6 +256,51 @@ def test_state_json(t : T?) { } } +[test] +def test_state_json_tts(t : T?) { + t |> run("the tts list: the front-end packs the route loads, then the models, each saying whether it reads them") @(t : T?) { + let dir = tmp_models_dir("ttsdoc") + let planted = path_join(dir, "pocket-tts-en-stuart-kq.gguf") + fopen(planted, "wb") $(f) { + if (f != null) { + f |> fwrite("GGUF") + } + } + var jerr = "" + var doc = read_json(catalog_state_json(dir, 0, 0l, @@needs_packs_by_name), jerr) + t |> success(doc != null, "state json parses: {jerr}") + let items = doc?["tts"] + t |> success(items != null && (items.value is _array), "tts is an array") + if (items != null && (items.value is _array)) { + var packs : array + var models = 0 + for (item in items.value as _array) { + let file = "{item?["file"] ?? ""}" + if (item?["pack"] ?? false) { + packs |> push(file) + t |> equal(item?["needs_packs"] ?? true, false, "{file}: a pack reads no pack") + } else { + models++ + let present = item?["present"] ?? false + t |> equal(present, file == "pocket-tts-en-stuart-kq.gguf", "{file}: presence follows the dir") + t |> equal(item?["needs_packs"] ?? false, !present, "{file}: an absent model reads as needing the packs, the planted Pocket file as not") + } + } + t |> equal(packs, TTS_FRONT_END_PACKS, "the pack rows are the route's own list, in its order - the browser's American twin rides no ladder") + var served = 0 + for (item in serve_tts_set()) { + served += item.pack ? 0 : 1 + } + t |> equal(models, served, "every served model is a row") + } + unsafe { + delete doc + } + remove(planted) + rmdir(dir) + } +} + [test] def test_fixture_path_normalization(t : T?) { t |> run("the captured catalog fixtures carry only normalized paths") @(t : T?) { diff --git a/utils/dasllama-server/tests/fixtures/catalog_done.json b/utils/dasllama-server/tests/fixtures/catalog_done.json index 5378d8b2e6..ac9fc3317e 100644 --- a/utils/dasllama-server/tests/fixtures/catalog_done.json +++ b/utils/dasllama-server/tests/fixtures/catalog_done.json @@ -1 +1 @@ -{"models_dir": "C:\\Users\\user\\.dasllama\\models", "entries": [{"is_default": true, "path": "C:\\Users\\user\\.dasllama\\models\\gemma-4-E2B-it-Q4_K_M.gguf", "sha256": "740185b21d22ceb83a11c3aa62ad5842ef32c70f6096d756bbee85a1e4ec34b8", "vision_bytes": 986833664, "display": "Gemma 4 E2B", "file": "gemma-4-E2B-it-Q4_K_M.gguf", "bytes": 3106738272, "present": true, "name": "gemma-4-e2b", "vram_hint_gb": 4, "vision_file": "mmproj-gemma-4-E2B-it-bf16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the recommended default - fast, capable, runs everywhere"}, {"is_default": false, "path": "", "sha256": "85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87", "vision_bytes": 991552256, "display": "Gemma 4 E4B", "file": "gemma-4-E4B-it-Q4_K_M.gguf", "bytes": 4977171584, "present": false, "name": "gemma-4-e4b", "vram_hint_gb": 6.5, "vision_file": "mmproj-gemma-4-E4B-it-BF16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the bigger E-series - better answers, still laptop-class"}, {"is_default": false, "path": "", "sha256": "f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f", "vision_bytes": 0, "display": "Gemma 4 26B-A4B", "file": "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "bytes": 16947541728, "present": false, "name": "gemma-4-26b-a4b", "vram_hint_gb": 19, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 131072, "note": "the MoE - 26B quality at 4B active weights per token"}, {"is_default": false, "path": "", "sha256": "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482", "vision_bytes": 0, "display": "Qwen 3.8 27B", "file": "Qwen3.8-27B-UD-Q4_K_M.gguf", "bytes": 16464440224, "present": false, "name": "qwen3.8-27b", "vram_hint_gb": 18.5, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the newest dense Qwen - thinking model, strong at code"}, {"is_default": false, "path": "", "sha256": "054721f478bc5fa6beffb7f38eae575d45298f88cbb8d2f83ef675a727863eb1", "vision_bytes": 836180256, "display": "Qwen3 VL 4B", "file": "Qwen3VL-4B-Instruct-Q8_0.gguf", "bytes": 4280406144, "present": false, "name": "qwen3-vl-4b", "vram_hint_gb": 6, "vision_file": "mmproj-Qwen3VL-4B-Instruct-F16.gguf", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the deepstack vision Qwen - reads images through wide multi-tap rows"}, {"is_default": false, "path": "", "sha256": "c7d8b07c8d8d7a9ed1de1b8df7ac821eb4d259a224bd44310baacfaa5a473d4c", "vision_bytes": 2623983328, "display": "Qwen2.5 Omni 3B", "file": "Qwen2.5-Omni-3B-Q8_0.gguf", "bytes": 3616087360, "present": false, "name": "qwen2.5-omni-3b", "vram_hint_gb": 5.5, "vision_file": "mmproj-Qwen2.5-Omni-3B-f16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the window-ViT omni - compact vision chat on the Metal tower"}, {"is_default": false, "path": "", "sha256": "0b21525e972670ed59e1812e170b27c26355381f0656ecc4e25617ece7dac58b", "vision_bytes": 0, "display": "Qwen 3.6 35B-A3B", "file": "Qwen3.6-35B-A3B-MTP-UD-Q4_K_M.gguf", "bytes": 22663387424, "present": false, "name": "qwen3.6-35b-a3b", "vram_hint_gb": 24, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the Qwen MoE - 3B active weights, MTP-ready"}], "asr": {"file": "ggml-parakeet-tdt-0.6b-v3-f32.bin", "bytes": 2508463079, "present": false, "path": ""}, "tts": [{"file": "kitten-nano.gguf", "bytes": 59331456, "pack": false, "present": false, "path": ""}, {"file": "tts_g2p.bin", "bytes": 14011554, "pack": true, "present": false, "path": ""}, {"file": "tts_postag.bin", "bytes": 12566510, "pack": true, "present": false, "path": ""}, {"file": "kitten-mini.gguf", "bytes": 295975008, "pack": false, "present": false, "path": ""}, {"file": "kokoro-82m.gguf", "bytes": 352965024, "pack": false, "present": false, "path": ""}], "box": {"ram_gb": 64, "vram_mb": 0}, "download": {"state": "done", "name": "gemma-4-e2b", "got": 0, "total": 3106738272, "error": ""}} \ No newline at end of file +{"models_dir": "C:\\Users\\user\\.dasllama\\models", "entries": [{"is_default": true, "path": "C:\\Users\\user\\.dasllama\\models\\gemma-4-E2B-it-Q4_K_M.gguf", "sha256": "740185b21d22ceb83a11c3aa62ad5842ef32c70f6096d756bbee85a1e4ec34b8", "vision_bytes": 986833664, "display": "Gemma 4 E2B", "file": "gemma-4-E2B-it-Q4_K_M.gguf", "bytes": 3106738272, "present": true, "name": "gemma-4-e2b", "vram_hint_gb": 4, "vision_file": "mmproj-gemma-4-E2B-it-bf16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the recommended default - fast, capable, runs everywhere"}, {"is_default": false, "path": "", "sha256": "85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87", "vision_bytes": 991552256, "display": "Gemma 4 E4B", "file": "gemma-4-E4B-it-Q4_K_M.gguf", "bytes": 4977171584, "present": false, "name": "gemma-4-e4b", "vram_hint_gb": 6.5, "vision_file": "mmproj-gemma-4-E4B-it-BF16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the bigger E-series - better answers, still laptop-class"}, {"is_default": false, "path": "", "sha256": "f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f", "vision_bytes": 0, "display": "Gemma 4 26B-A4B", "file": "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "bytes": 16947541728, "present": false, "name": "gemma-4-26b-a4b", "vram_hint_gb": 19, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 131072, "note": "the MoE - 26B quality at 4B active weights per token"}, {"is_default": false, "path": "", "sha256": "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482", "vision_bytes": 0, "display": "Qwen 3.8 27B", "file": "Qwen3.8-27B-UD-Q4_K_M.gguf", "bytes": 16464440224, "present": false, "name": "qwen3.8-27b", "vram_hint_gb": 18.5, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the newest dense Qwen - thinking model, strong at code"}, {"is_default": false, "path": "", "sha256": "054721f478bc5fa6beffb7f38eae575d45298f88cbb8d2f83ef675a727863eb1", "vision_bytes": 836180256, "display": "Qwen3 VL 4B", "file": "Qwen3VL-4B-Instruct-Q8_0.gguf", "bytes": 4280406144, "present": false, "name": "qwen3-vl-4b", "vram_hint_gb": 6, "vision_file": "mmproj-Qwen3VL-4B-Instruct-F16.gguf", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the deepstack vision Qwen - reads images through wide multi-tap rows"}, {"is_default": false, "path": "", "sha256": "c7d8b07c8d8d7a9ed1de1b8df7ac821eb4d259a224bd44310baacfaa5a473d4c", "vision_bytes": 2623983328, "display": "Qwen2.5 Omni 3B", "file": "Qwen2.5-Omni-3B-Q8_0.gguf", "bytes": 3616087360, "present": false, "name": "qwen2.5-omni-3b", "vram_hint_gb": 5.5, "vision_file": "mmproj-Qwen2.5-Omni-3B-f16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the window-ViT omni - compact vision chat on the Metal tower"}, {"is_default": false, "path": "", "sha256": "0b21525e972670ed59e1812e170b27c26355381f0656ecc4e25617ece7dac58b", "vision_bytes": 0, "display": "Qwen 3.6 35B-A3B", "file": "Qwen3.6-35B-A3B-MTP-UD-Q4_K_M.gguf", "bytes": 22663387424, "present": false, "name": "qwen3.6-35b-a3b", "vram_hint_gb": 24, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the Qwen MoE - 3B active weights, MTP-ready"}], "asr": {"file": "ggml-parakeet-tdt-0.6b-v3-f32.bin", "bytes": 2508463079, "present": false, "path": ""}, "tts": [{"file": "kitten-nano.gguf", "bytes": 59331456, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "tts_g2p.bin", "bytes": 14011554, "pack": true, "present": false, "path": "", "needs_packs": false}, {"file": "tts_postag.bin", "bytes": 12566510, "pack": true, "present": false, "path": "", "needs_packs": false}, {"file": "kitten-mini.gguf", "bytes": 295975008, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "kokoro-82m.gguf", "bytes": 352965024, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-q8.gguf", "bytes": 152613664, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-de-q8.gguf", "bytes": 134667200, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-es-q8.gguf", "bytes": 134624480, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-it-q8.gguf", "bytes": 134415072, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-pt-q8.gguf", "bytes": 134667488, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-fr-q8.gguf", "bytes": 375793696, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-kq.gguf", "bytes": 74970016, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-stuart-kq.gguf", "bytes": 65107520, "pack": false, "present": false, "path": "", "needs_packs": true}], "box": {"ram_gb": 64, "vram_mb": 0}, "download": {"state": "done", "name": "gemma-4-e2b", "got": 0, "total": 3106738272, "error": ""}} \ No newline at end of file diff --git a/utils/dasllama-server/tests/fixtures/catalog_downloading.json b/utils/dasllama-server/tests/fixtures/catalog_downloading.json index e34da76038..e820e54050 100644 --- a/utils/dasllama-server/tests/fixtures/catalog_downloading.json +++ b/utils/dasllama-server/tests/fixtures/catalog_downloading.json @@ -1 +1 @@ -{"models_dir": "C:\\Users\\user\\.dasllama\\models", "entries": [{"is_default": true, "path": "", "sha256": "740185b21d22ceb83a11c3aa62ad5842ef32c70f6096d756bbee85a1e4ec34b8", "vision_bytes": 986833664, "display": "Gemma 4 E2B", "file": "gemma-4-E2B-it-Q4_K_M.gguf", "bytes": 3106738272, "present": false, "name": "gemma-4-e2b", "vram_hint_gb": 4, "vision_file": "mmproj-gemma-4-E2B-it-bf16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the recommended default - fast, capable, runs everywhere"}, {"is_default": false, "path": "", "sha256": "85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87", "vision_bytes": 991552256, "display": "Gemma 4 E4B", "file": "gemma-4-E4B-it-Q4_K_M.gguf", "bytes": 4977171584, "present": false, "name": "gemma-4-e4b", "vram_hint_gb": 6.5, "vision_file": "mmproj-gemma-4-E4B-it-BF16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the bigger E-series - better answers, still laptop-class"}, {"is_default": false, "path": "", "sha256": "f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f", "vision_bytes": 0, "display": "Gemma 4 26B-A4B", "file": "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "bytes": 16947541728, "present": false, "name": "gemma-4-26b-a4b", "vram_hint_gb": 19, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 131072, "note": "the MoE - 26B quality at 4B active weights per token"}, {"is_default": false, "path": "", "sha256": "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482", "vision_bytes": 0, "display": "Qwen 3.8 27B", "file": "Qwen3.8-27B-UD-Q4_K_M.gguf", "bytes": 16464440224, "present": false, "name": "qwen3.8-27b", "vram_hint_gb": 18.5, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the newest dense Qwen - thinking model, strong at code"}, {"is_default": false, "path": "", "sha256": "054721f478bc5fa6beffb7f38eae575d45298f88cbb8d2f83ef675a727863eb1", "vision_bytes": 836180256, "display": "Qwen3 VL 4B", "file": "Qwen3VL-4B-Instruct-Q8_0.gguf", "bytes": 4280406144, "present": false, "name": "qwen3-vl-4b", "vram_hint_gb": 6, "vision_file": "mmproj-Qwen3VL-4B-Instruct-F16.gguf", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the deepstack vision Qwen - reads images through wide multi-tap rows"}, {"is_default": false, "path": "", "sha256": "c7d8b07c8d8d7a9ed1de1b8df7ac821eb4d259a224bd44310baacfaa5a473d4c", "vision_bytes": 2623983328, "display": "Qwen2.5 Omni 3B", "file": "Qwen2.5-Omni-3B-Q8_0.gguf", "bytes": 3616087360, "present": false, "name": "qwen2.5-omni-3b", "vram_hint_gb": 5.5, "vision_file": "mmproj-Qwen2.5-Omni-3B-f16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the window-ViT omni - compact vision chat on the Metal tower"}, {"is_default": false, "path": "", "sha256": "0b21525e972670ed59e1812e170b27c26355381f0656ecc4e25617ece7dac58b", "vision_bytes": 0, "display": "Qwen 3.6 35B-A3B", "file": "Qwen3.6-35B-A3B-MTP-UD-Q4_K_M.gguf", "bytes": 22663387424, "present": false, "name": "qwen3.6-35b-a3b", "vram_hint_gb": 24, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the Qwen MoE - 3B active weights, MTP-ready"}], "asr": {"file": "ggml-parakeet-tdt-0.6b-v3-f32.bin", "bytes": 2508463079, "present": false, "path": ""}, "tts": [{"file": "kitten-nano.gguf", "bytes": 59331456, "pack": false, "present": false, "path": ""}, {"file": "tts_g2p.bin", "bytes": 14011554, "pack": true, "present": false, "path": ""}, {"file": "tts_postag.bin", "bytes": 12566510, "pack": true, "present": false, "path": ""}, {"file": "kitten-mini.gguf", "bytes": 295975008, "pack": false, "present": false, "path": ""}, {"file": "kokoro-82m.gguf", "bytes": 352965024, "pack": false, "present": false, "path": ""}], "box": {"ram_gb": 64, "vram_mb": 0}, "download": {"state": "downloading", "name": "gemma-4-e2b", "got": 157736960, "total": 3106738272, "error": ""}} \ No newline at end of file +{"models_dir": "C:\\Users\\user\\.dasllama\\models", "entries": [{"is_default": true, "path": "", "sha256": "740185b21d22ceb83a11c3aa62ad5842ef32c70f6096d756bbee85a1e4ec34b8", "vision_bytes": 986833664, "display": "Gemma 4 E2B", "file": "gemma-4-E2B-it-Q4_K_M.gguf", "bytes": 3106738272, "present": false, "name": "gemma-4-e2b", "vram_hint_gb": 4, "vision_file": "mmproj-gemma-4-E2B-it-bf16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the recommended default - fast, capable, runs everywhere"}, {"is_default": false, "path": "", "sha256": "85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87", "vision_bytes": 991552256, "display": "Gemma 4 E4B", "file": "gemma-4-E4B-it-Q4_K_M.gguf", "bytes": 4977171584, "present": false, "name": "gemma-4-e4b", "vram_hint_gb": 6.5, "vision_file": "mmproj-gemma-4-E4B-it-BF16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the bigger E-series - better answers, still laptop-class"}, {"is_default": false, "path": "", "sha256": "f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f", "vision_bytes": 0, "display": "Gemma 4 26B-A4B", "file": "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "bytes": 16947541728, "present": false, "name": "gemma-4-26b-a4b", "vram_hint_gb": 19, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 131072, "note": "the MoE - 26B quality at 4B active weights per token"}, {"is_default": false, "path": "", "sha256": "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482", "vision_bytes": 0, "display": "Qwen 3.8 27B", "file": "Qwen3.8-27B-UD-Q4_K_M.gguf", "bytes": 16464440224, "present": false, "name": "qwen3.8-27b", "vram_hint_gb": 18.5, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the newest dense Qwen - thinking model, strong at code"}, {"is_default": false, "path": "", "sha256": "054721f478bc5fa6beffb7f38eae575d45298f88cbb8d2f83ef675a727863eb1", "vision_bytes": 836180256, "display": "Qwen3 VL 4B", "file": "Qwen3VL-4B-Instruct-Q8_0.gguf", "bytes": 4280406144, "present": false, "name": "qwen3-vl-4b", "vram_hint_gb": 6, "vision_file": "mmproj-Qwen3VL-4B-Instruct-F16.gguf", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the deepstack vision Qwen - reads images through wide multi-tap rows"}, {"is_default": false, "path": "", "sha256": "c7d8b07c8d8d7a9ed1de1b8df7ac821eb4d259a224bd44310baacfaa5a473d4c", "vision_bytes": 2623983328, "display": "Qwen2.5 Omni 3B", "file": "Qwen2.5-Omni-3B-Q8_0.gguf", "bytes": 3616087360, "present": false, "name": "qwen2.5-omni-3b", "vram_hint_gb": 5.5, "vision_file": "mmproj-Qwen2.5-Omni-3B-f16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the window-ViT omni - compact vision chat on the Metal tower"}, {"is_default": false, "path": "", "sha256": "0b21525e972670ed59e1812e170b27c26355381f0656ecc4e25617ece7dac58b", "vision_bytes": 0, "display": "Qwen 3.6 35B-A3B", "file": "Qwen3.6-35B-A3B-MTP-UD-Q4_K_M.gguf", "bytes": 22663387424, "present": false, "name": "qwen3.6-35b-a3b", "vram_hint_gb": 24, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the Qwen MoE - 3B active weights, MTP-ready"}], "asr": {"file": "ggml-parakeet-tdt-0.6b-v3-f32.bin", "bytes": 2508463079, "present": false, "path": ""}, "tts": [{"file": "kitten-nano.gguf", "bytes": 59331456, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "tts_g2p.bin", "bytes": 14011554, "pack": true, "present": false, "path": "", "needs_packs": false}, {"file": "tts_postag.bin", "bytes": 12566510, "pack": true, "present": false, "path": "", "needs_packs": false}, {"file": "kitten-mini.gguf", "bytes": 295975008, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "kokoro-82m.gguf", "bytes": 352965024, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-q8.gguf", "bytes": 152613664, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-de-q8.gguf", "bytes": 134667200, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-es-q8.gguf", "bytes": 134624480, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-it-q8.gguf", "bytes": 134415072, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-pt-q8.gguf", "bytes": 134667488, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-fr-q8.gguf", "bytes": 375793696, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-kq.gguf", "bytes": 74970016, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-stuart-kq.gguf", "bytes": 65107520, "pack": false, "present": false, "path": "", "needs_packs": true}], "box": {"ram_gb": 64, "vram_mb": 0}, "download": {"state": "downloading", "name": "gemma-4-e2b", "got": 160342016, "total": 3106738272, "error": ""}} \ No newline at end of file diff --git a/utils/dasllama-server/tests/fixtures/catalog_empty.json b/utils/dasllama-server/tests/fixtures/catalog_empty.json index 10f183648c..4b06409c8e 100644 --- a/utils/dasllama-server/tests/fixtures/catalog_empty.json +++ b/utils/dasllama-server/tests/fixtures/catalog_empty.json @@ -1 +1 @@ -{"models_dir": "C:\\Users\\user\\.dasllama\\models", "entries": [{"is_default": true, "path": "", "sha256": "740185b21d22ceb83a11c3aa62ad5842ef32c70f6096d756bbee85a1e4ec34b8", "vision_bytes": 986833664, "display": "Gemma 4 E2B", "file": "gemma-4-E2B-it-Q4_K_M.gguf", "bytes": 3106738272, "present": false, "name": "gemma-4-e2b", "vram_hint_gb": 4, "vision_file": "mmproj-gemma-4-E2B-it-bf16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the recommended default - fast, capable, runs everywhere"}, {"is_default": false, "path": "", "sha256": "85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87", "vision_bytes": 991552256, "display": "Gemma 4 E4B", "file": "gemma-4-E4B-it-Q4_K_M.gguf", "bytes": 4977171584, "present": false, "name": "gemma-4-e4b", "vram_hint_gb": 6.5, "vision_file": "mmproj-gemma-4-E4B-it-BF16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the bigger E-series - better answers, still laptop-class"}, {"is_default": false, "path": "", "sha256": "f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f", "vision_bytes": 0, "display": "Gemma 4 26B-A4B", "file": "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "bytes": 16947541728, "present": false, "name": "gemma-4-26b-a4b", "vram_hint_gb": 19, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 131072, "note": "the MoE - 26B quality at 4B active weights per token"}, {"is_default": false, "path": "", "sha256": "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482", "vision_bytes": 0, "display": "Qwen 3.8 27B", "file": "Qwen3.8-27B-UD-Q4_K_M.gguf", "bytes": 16464440224, "present": false, "name": "qwen3.8-27b", "vram_hint_gb": 18.5, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the newest dense Qwen - thinking model, strong at code"}, {"is_default": false, "path": "", "sha256": "054721f478bc5fa6beffb7f38eae575d45298f88cbb8d2f83ef675a727863eb1", "vision_bytes": 836180256, "display": "Qwen3 VL 4B", "file": "Qwen3VL-4B-Instruct-Q8_0.gguf", "bytes": 4280406144, "present": false, "name": "qwen3-vl-4b", "vram_hint_gb": 6, "vision_file": "mmproj-Qwen3VL-4B-Instruct-F16.gguf", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the deepstack vision Qwen - reads images through wide multi-tap rows"}, {"is_default": false, "path": "", "sha256": "c7d8b07c8d8d7a9ed1de1b8df7ac821eb4d259a224bd44310baacfaa5a473d4c", "vision_bytes": 2623983328, "display": "Qwen2.5 Omni 3B", "file": "Qwen2.5-Omni-3B-Q8_0.gguf", "bytes": 3616087360, "present": false, "name": "qwen2.5-omni-3b", "vram_hint_gb": 5.5, "vision_file": "mmproj-Qwen2.5-Omni-3B-f16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the window-ViT omni - compact vision chat on the Metal tower"}, {"is_default": false, "path": "", "sha256": "0b21525e972670ed59e1812e170b27c26355381f0656ecc4e25617ece7dac58b", "vision_bytes": 0, "display": "Qwen 3.6 35B-A3B", "file": "Qwen3.6-35B-A3B-MTP-UD-Q4_K_M.gguf", "bytes": 22663387424, "present": false, "name": "qwen3.6-35b-a3b", "vram_hint_gb": 24, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the Qwen MoE - 3B active weights, MTP-ready"}], "asr": {"file": "ggml-parakeet-tdt-0.6b-v3-f32.bin", "bytes": 2508463079, "present": false, "path": ""}, "tts": [{"file": "kitten-nano.gguf", "bytes": 59331456, "pack": false, "present": false, "path": ""}, {"file": "tts_g2p.bin", "bytes": 14011554, "pack": true, "present": false, "path": ""}, {"file": "tts_postag.bin", "bytes": 12566510, "pack": true, "present": false, "path": ""}, {"file": "kitten-mini.gguf", "bytes": 295975008, "pack": false, "present": false, "path": ""}, {"file": "kokoro-82m.gguf", "bytes": 352965024, "pack": false, "present": false, "path": ""}], "box": {"ram_gb": 64, "vram_mb": 0}, "download": {"state": "idle", "name": "", "got": 0, "total": 0, "error": ""}} \ No newline at end of file +{"models_dir": "C:\\Users\\user\\.dasllama\\models", "entries": [{"is_default": true, "path": "", "sha256": "740185b21d22ceb83a11c3aa62ad5842ef32c70f6096d756bbee85a1e4ec34b8", "vision_bytes": 986833664, "display": "Gemma 4 E2B", "file": "gemma-4-E2B-it-Q4_K_M.gguf", "bytes": 3106738272, "present": false, "name": "gemma-4-e2b", "vram_hint_gb": 4, "vision_file": "mmproj-gemma-4-E2B-it-bf16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the recommended default - fast, capable, runs everywhere"}, {"is_default": false, "path": "", "sha256": "85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87", "vision_bytes": 991552256, "display": "Gemma 4 E4B", "file": "gemma-4-E4B-it-Q4_K_M.gguf", "bytes": 4977171584, "present": false, "name": "gemma-4-e4b", "vram_hint_gb": 6.5, "vision_file": "mmproj-gemma-4-E4B-it-BF16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the bigger E-series - better answers, still laptop-class"}, {"is_default": false, "path": "", "sha256": "f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f", "vision_bytes": 0, "display": "Gemma 4 26B-A4B", "file": "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "bytes": 16947541728, "present": false, "name": "gemma-4-26b-a4b", "vram_hint_gb": 19, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 131072, "note": "the MoE - 26B quality at 4B active weights per token"}, {"is_default": false, "path": "", "sha256": "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482", "vision_bytes": 0, "display": "Qwen 3.8 27B", "file": "Qwen3.8-27B-UD-Q4_K_M.gguf", "bytes": 16464440224, "present": false, "name": "qwen3.8-27b", "vram_hint_gb": 18.5, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the newest dense Qwen - thinking model, strong at code"}, {"is_default": false, "path": "", "sha256": "054721f478bc5fa6beffb7f38eae575d45298f88cbb8d2f83ef675a727863eb1", "vision_bytes": 836180256, "display": "Qwen3 VL 4B", "file": "Qwen3VL-4B-Instruct-Q8_0.gguf", "bytes": 4280406144, "present": false, "name": "qwen3-vl-4b", "vram_hint_gb": 6, "vision_file": "mmproj-Qwen3VL-4B-Instruct-F16.gguf", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the deepstack vision Qwen - reads images through wide multi-tap rows"}, {"is_default": false, "path": "", "sha256": "c7d8b07c8d8d7a9ed1de1b8df7ac821eb4d259a224bd44310baacfaa5a473d4c", "vision_bytes": 2623983328, "display": "Qwen2.5 Omni 3B", "file": "Qwen2.5-Omni-3B-Q8_0.gguf", "bytes": 3616087360, "present": false, "name": "qwen2.5-omni-3b", "vram_hint_gb": 5.5, "vision_file": "mmproj-Qwen2.5-Omni-3B-f16.gguf", "vision_present": false, "vision_path": "", "ctx": 32768, "note": "the window-ViT omni - compact vision chat on the Metal tower"}, {"is_default": false, "path": "", "sha256": "0b21525e972670ed59e1812e170b27c26355381f0656ecc4e25617ece7dac58b", "vision_bytes": 0, "display": "Qwen 3.6 35B-A3B", "file": "Qwen3.6-35B-A3B-MTP-UD-Q4_K_M.gguf", "bytes": 22663387424, "present": false, "name": "qwen3.6-35b-a3b", "vram_hint_gb": 24, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the Qwen MoE - 3B active weights, MTP-ready"}], "asr": {"file": "ggml-parakeet-tdt-0.6b-v3-f32.bin", "bytes": 2508463079, "present": false, "path": ""}, "tts": [{"file": "kitten-nano.gguf", "bytes": 59331456, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "tts_g2p.bin", "bytes": 14011554, "pack": true, "present": false, "path": "", "needs_packs": false}, {"file": "tts_postag.bin", "bytes": 12566510, "pack": true, "present": false, "path": "", "needs_packs": false}, {"file": "kitten-mini.gguf", "bytes": 295975008, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "kokoro-82m.gguf", "bytes": 352965024, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-q8.gguf", "bytes": 152613664, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-de-q8.gguf", "bytes": 134667200, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-es-q8.gguf", "bytes": 134624480, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-it-q8.gguf", "bytes": 134415072, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-pt-q8.gguf", "bytes": 134667488, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-fr-q8.gguf", "bytes": 375793696, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-kq.gguf", "bytes": 74970016, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-stuart-kq.gguf", "bytes": 65107520, "pack": false, "present": false, "path": "", "needs_packs": true}], "box": {"ram_gb": 64, "vram_mb": 0}, "download": {"state": "idle", "name": "", "got": 0, "total": 0, "error": ""}} \ No newline at end of file diff --git a/utils/dasllama-server/tests/fixtures/catalog_idle.json b/utils/dasllama-server/tests/fixtures/catalog_idle.json index 7af982ab03..d0e11c62a6 100644 --- a/utils/dasllama-server/tests/fixtures/catalog_idle.json +++ b/utils/dasllama-server/tests/fixtures/catalog_idle.json @@ -1 +1 @@ -{"models_dir": "C:\\Users\\user\\.dasllama\\models", "entries": [{"is_default": true, "path": "C:\\Users\\user\\.dasllama\\models\\gemma-4-E2B-it-Q4_K_M.gguf", "sha256": "740185b21d22ceb83a11c3aa62ad5842ef32c70f6096d756bbee85a1e4ec34b8", "vision_bytes": 986833664, "display": "Gemma 4 E2B", "file": "gemma-4-E2B-it-Q4_K_M.gguf", "bytes": 3106738272, "present": true, "name": "gemma-4-e2b", "vram_hint_gb": 4, "vision_file": "mmproj-gemma-4-E2B-it-bf16.gguf", "vision_present": true, "vision_path": "C:\\Users\\user\\.dasllama\\models\\mmproj-gemma-4-E2B-it-bf16.gguf", "ctx": 32768, "note": "the recommended default - fast, capable, runs everywhere"}, {"is_default": false, "path": "", "sha256": "85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87", "vision_bytes": 991552256, "display": "Gemma 4 E4B", "file": "gemma-4-E4B-it-Q4_K_M.gguf", "bytes": 4977171584, "present": false, "name": "gemma-4-e4b", "vram_hint_gb": 6.5, "vision_file": "mmproj-gemma-4-E4B-it-BF16.gguf", "vision_present": true, "vision_path": "C:\\Users\\user\\.dasllama\\models\\mmproj-gemma-4-E4B-it-BF16.gguf", "ctx": 32768, "note": "the bigger E-series - better answers, still laptop-class"}, {"is_default": false, "path": "", "sha256": "f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f", "vision_bytes": 0, "display": "Gemma 4 26B-A4B", "file": "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "bytes": 16947541728, "present": false, "name": "gemma-4-26b-a4b", "vram_hint_gb": 19, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 131072, "note": "the MoE - 26B quality at 4B active weights per token"}, {"is_default": false, "path": "", "sha256": "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482", "vision_bytes": 0, "display": "Qwen 3.8 27B", "file": "Qwen3.8-27B-UD-Q4_K_M.gguf", "bytes": 16464440224, "present": false, "name": "qwen3.8-27b", "vram_hint_gb": 18.5, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the newest dense Qwen - thinking model, strong at code"}, {"is_default": false, "path": "C:\\Users\\user\\.dasllama\\models\\Qwen3VL-4B-Instruct-Q8_0.gguf", "sha256": "054721f478bc5fa6beffb7f38eae575d45298f88cbb8d2f83ef675a727863eb1", "vision_bytes": 836180256, "display": "Qwen3 VL 4B", "file": "Qwen3VL-4B-Instruct-Q8_0.gguf", "bytes": 4280406144, "present": true, "name": "qwen3-vl-4b", "vram_hint_gb": 6, "vision_file": "mmproj-Qwen3VL-4B-Instruct-F16.gguf", "vision_present": true, "vision_path": "C:\\Users\\user\\.dasllama\\models\\mmproj-Qwen3VL-4B-Instruct-F16.gguf", "ctx": 262144, "note": "the deepstack vision Qwen - reads images through wide multi-tap rows"}, {"is_default": false, "path": "C:\\Users\\user\\.dasllama\\models\\Qwen2.5-Omni-3B-Q8_0.gguf", "sha256": "c7d8b07c8d8d7a9ed1de1b8df7ac821eb4d259a224bd44310baacfaa5a473d4c", "vision_bytes": 2623983328, "display": "Qwen2.5 Omni 3B", "file": "Qwen2.5-Omni-3B-Q8_0.gguf", "bytes": 3616087360, "present": true, "name": "qwen2.5-omni-3b", "vram_hint_gb": 5.5, "vision_file": "mmproj-Qwen2.5-Omni-3B-f16.gguf", "vision_present": true, "vision_path": "C:\\Users\\user\\.dasllama\\models\\mmproj-Qwen2.5-Omni-3B-f16.gguf", "ctx": 32768, "note": "the window-ViT omni - compact vision chat on the Metal tower"}, {"is_default": false, "path": "C:\\Users\\user\\.dasllama\\models\\Qwen3.6-35B-A3B-MTP-UD-Q4_K_M.gguf", "sha256": "0b21525e972670ed59e1812e170b27c26355381f0656ecc4e25617ece7dac58b", "vision_bytes": 0, "display": "Qwen 3.6 35B-A3B", "file": "Qwen3.6-35B-A3B-MTP-UD-Q4_K_M.gguf", "bytes": 22663387424, "present": true, "name": "qwen3.6-35b-a3b", "vram_hint_gb": 24, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the Qwen MoE - 3B active weights, MTP-ready"}], "asr": {"file": "ggml-parakeet-tdt-0.6b-v3-f32.bin", "bytes": 2508463079, "present": false, "path": ""}, "tts": [{"file": "kitten-nano.gguf", "bytes": 59331456, "pack": false, "present": true, "path": "C:\\Users\\user\\.dasllama\\models\\kitten-nano.gguf"}, {"file": "tts_g2p.bin", "bytes": 14011554, "pack": true, "present": true, "path": "C:\\Users\\user\\.dasllama\\models\\tts_g2p.bin"}, {"file": "tts_postag.bin", "bytes": 12566510, "pack": true, "present": true, "path": "C:\\Users\\user\\.dasllama\\models\\tts_postag.bin"}, {"file": "kitten-mini.gguf", "bytes": 295975008, "pack": false, "present": false, "path": ""}, {"file": "kokoro-82m.gguf", "bytes": 352965024, "pack": false, "present": false, "path": ""}], "box": {"ram_gb": 64, "vram_mb": 0}, "download": {"state": "idle", "name": "", "got": 0, "total": 0, "error": ""}} \ No newline at end of file +{"models_dir": "C:\\Users\\user\\.dasllama\\models", "entries": [{"is_default": true, "path": "C:\\Users\\user\\.dasllama\\models\\gemma-4-E2B-it-Q4_K_M.gguf", "sha256": "740185b21d22ceb83a11c3aa62ad5842ef32c70f6096d756bbee85a1e4ec34b8", "vision_bytes": 986833664, "display": "Gemma 4 E2B", "file": "gemma-4-E2B-it-Q4_K_M.gguf", "bytes": 3106738272, "present": true, "name": "gemma-4-e2b", "vram_hint_gb": 4, "vision_file": "mmproj-gemma-4-E2B-it-bf16.gguf", "vision_present": true, "vision_path": "C:\\Users\\user\\.dasllama\\models\\mmproj-gemma-4-E2B-it-bf16.gguf", "ctx": 32768, "note": "the recommended default - fast, capable, runs everywhere"}, {"is_default": false, "path": "", "sha256": "85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87", "vision_bytes": 991552256, "display": "Gemma 4 E4B", "file": "gemma-4-E4B-it-Q4_K_M.gguf", "bytes": 4977171584, "present": false, "name": "gemma-4-e4b", "vram_hint_gb": 6.5, "vision_file": "mmproj-gemma-4-E4B-it-BF16.gguf", "vision_present": true, "vision_path": "C:\\Users\\user\\.dasllama\\models\\mmproj-gemma-4-E4B-it-BF16.gguf", "ctx": 32768, "note": "the bigger E-series - better answers, still laptop-class"}, {"is_default": false, "path": "", "sha256": "f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f", "vision_bytes": 0, "display": "Gemma 4 26B-A4B", "file": "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "bytes": 16947541728, "present": false, "name": "gemma-4-26b-a4b", "vram_hint_gb": 19, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 131072, "note": "the MoE - 26B quality at 4B active weights per token"}, {"is_default": false, "path": "", "sha256": "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482", "vision_bytes": 0, "display": "Qwen 3.8 27B", "file": "Qwen3.8-27B-UD-Q4_K_M.gguf", "bytes": 16464440224, "present": false, "name": "qwen3.8-27b", "vram_hint_gb": 18.5, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the newest dense Qwen - thinking model, strong at code"}, {"is_default": false, "path": "C:\\Users\\user\\.dasllama\\models\\Qwen3VL-4B-Instruct-Q8_0.gguf", "sha256": "054721f478bc5fa6beffb7f38eae575d45298f88cbb8d2f83ef675a727863eb1", "vision_bytes": 836180256, "display": "Qwen3 VL 4B", "file": "Qwen3VL-4B-Instruct-Q8_0.gguf", "bytes": 4280406144, "present": true, "name": "qwen3-vl-4b", "vram_hint_gb": 6, "vision_file": "mmproj-Qwen3VL-4B-Instruct-F16.gguf", "vision_present": true, "vision_path": "C:\\Users\\user\\.dasllama\\models\\mmproj-Qwen3VL-4B-Instruct-F16.gguf", "ctx": 262144, "note": "the deepstack vision Qwen - reads images through wide multi-tap rows"}, {"is_default": false, "path": "C:\\Users\\user\\.dasllama\\models\\Qwen2.5-Omni-3B-Q8_0.gguf", "sha256": "c7d8b07c8d8d7a9ed1de1b8df7ac821eb4d259a224bd44310baacfaa5a473d4c", "vision_bytes": 2623983328, "display": "Qwen2.5 Omni 3B", "file": "Qwen2.5-Omni-3B-Q8_0.gguf", "bytes": 3616087360, "present": true, "name": "qwen2.5-omni-3b", "vram_hint_gb": 5.5, "vision_file": "mmproj-Qwen2.5-Omni-3B-f16.gguf", "vision_present": true, "vision_path": "C:\\Users\\user\\.dasllama\\models\\mmproj-Qwen2.5-Omni-3B-f16.gguf", "ctx": 32768, "note": "the window-ViT omni - compact vision chat on the Metal tower"}, {"is_default": false, "path": "C:\\Users\\user\\.dasllama\\models\\Qwen3.6-35B-A3B-MTP-UD-Q4_K_M.gguf", "sha256": "0b21525e972670ed59e1812e170b27c26355381f0656ecc4e25617ece7dac58b", "vision_bytes": 0, "display": "Qwen 3.6 35B-A3B", "file": "Qwen3.6-35B-A3B-MTP-UD-Q4_K_M.gguf", "bytes": 22663387424, "present": true, "name": "qwen3.6-35b-a3b", "vram_hint_gb": 24, "vision_file": "", "vision_present": false, "vision_path": "", "ctx": 262144, "note": "the Qwen MoE - 3B active weights, MTP-ready"}], "asr": {"file": "ggml-parakeet-tdt-0.6b-v3-f32.bin", "bytes": 2508463079, "present": false, "path": ""}, "tts": [{"file": "kitten-nano.gguf", "bytes": 59331456, "pack": false, "present": true, "path": "C:\\Users\\user\\.dasllama\\models\\kitten-nano.gguf", "needs_packs": true}, {"file": "tts_g2p.bin", "bytes": 14011554, "pack": true, "present": true, "path": "C:\\Users\\user\\.dasllama\\models\\tts_g2p.bin", "needs_packs": false}, {"file": "tts_postag.bin", "bytes": 12566510, "pack": true, "present": true, "path": "C:\\Users\\user\\.dasllama\\models\\tts_postag.bin", "needs_packs": false}, {"file": "kitten-mini.gguf", "bytes": 295975008, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "kokoro-82m.gguf", "bytes": 352965024, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-q8.gguf", "bytes": 152613664, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-de-q8.gguf", "bytes": 134667200, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-es-q8.gguf", "bytes": 134624480, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-it-q8.gguf", "bytes": 134415072, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-pt-q8.gguf", "bytes": 134667488, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-fr-q8.gguf", "bytes": 375793696, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-kq.gguf", "bytes": 74970016, "pack": false, "present": false, "path": "", "needs_packs": true}, {"file": "pocket-tts-en-stuart-kq.gguf", "bytes": 65107520, "pack": false, "present": false, "path": "", "needs_packs": true}], "box": {"ram_gb": 64, "vram_mb": 0}, "download": {"state": "idle", "name": "", "got": 0, "total": 0, "error": ""}} \ No newline at end of file diff --git a/utils/dasllama-server/tests/tts.spec.js b/utils/dasllama-server/tests/tts.spec.js index 01f2b33233..314033d124 100644 --- a/utils/dasllama-server/tests/tts.spec.js +++ b/utils/dasllama-server/tests/tts.spec.js @@ -323,6 +323,37 @@ test('a downloaded model beside the packs offers enable-speech, which wires and expect(posts.some(p => p.path === '/restart')).toBe(true); }); +// the packs taken away again - the state a box is in with one file on disk +function withPacksAbsent(doc) { + const d = JSON.parse(JSON.stringify(doc)); + for (const i of d.tts) { + if (i.pack) { i.present = false; i.path = ''; } + } + return d; +} + +test('a model that reads no packs enables speech with the packs absent; one that reads them waits for them', async ({ page }) => { + const idle = fx('catalog_idle'); + const readsPacks = speechModels(idle).find(m => m.present); + expect(readsPacks.needs_packs).toBe(true); // the capture stocks one phoneme family + // the phoneme family alone, packs gone: the ladder re-offers the packs, no enable + const waiting = withPacksAbsent(idle); + await openControl(page, { catalog: waiting }); + await expect(page.locator('#tts-offer button', { hasText: 'enable speech' })).toHaveCount(0); + await expect(page.locator('#tts-offer button', { hasText: 'download the front-end packs' })).toHaveCount(1); + // a second model on disk whose row says it reads no packs - the page reads only that key, + // so any absent row of the capture serves, its needs_packs overridden: it is the one enable + // wires, the packs still absent + const alone = speechModels(idle).find(m => !m.present); + const doc = withModelPresent(waiting, alone.file); + doc.tts.find(i => i.file === alone.file).needs_packs = false; + const { posts } = await openControl(page, { catalog: doc }); + await expect(page.locator('#tts-pick')).toHaveCount(0); // the phoneme family is not a choice without its packs + await page.locator('#tts-offer button', { hasText: 'enable speech' }).click(); + await expect(page.locator('#cat-note')).toContainText('speech wired (' + modelName(alone.file) + ')'); + expect(lastJson(posts.filter(p => p.path === '/config')).tts).toBe(doc.models_dir + '\\' + alone.file); +}); + test('several downloaded models become a picker that defaults to the smallest', async ({ page }) => { const idle = fx('catalog_idle'); const absent = bySize(speechModels(idle).filter(m => !m.present))[0];