From 426716a4bb6399d9f780e3a8dbd7281ea3aaec85 Mon Sep 17 00:00:00 2001 From: Nerual Dreming Date: Thu, 12 Nov 2026 13:29:59 +0300 Subject: [PATCH] Shrink a stage's budget to the room the KV cache leaves instead of refusing the request; report a truncated track in its replay JSON --- README.md | 3 ++- docs/ARCHITECTURE.md | 11 +++++++---- src/generate.h | 21 ++++++++++++++++----- tools/yue-server.cpp | 22 +++++++++++++++++++++- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 7e8cac6..2c7d111 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,8 @@ jobs in FIFO order. **GET /job?id=N** - Poll job status. **GET /job?id=N&result=1** fetches the result as multipart/mixed, one pair per track: a JSON replay request part (the request carrying the semantic stream, the score and the seeds of the -track) then the audio part (MP3 or WAV, selected by `output_format` in the +track, plus `"truncated": true` when a stage hit its budget before its end +token) then the audio part (MP3 or WAV, selected by `output_format` in the request). `lm_batch_size` songs times `synth_batch_size` noise variations come out song-major. **POST /job?id=N&cancel=1** cancels a running job. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2a35aac..e7a01f0 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -172,9 +172,11 @@ free guidance, grown on demand; `--max-seq` and `--max-batch` are the levers that trade context and batch for VRAM. Measured peak above idle on a 65 s song in Q8_0 under STRICT: 5.8 GB at the full context, 3.8 GB at `--max-seq 8192`, the compute buffers of the three graphs making the -difference with the weights plus the cache, the VAE tiles first. A cache smaller than the -default budgets needs a `duration` on the request, the generator refuses -a prefix plus budget that would not fit. +difference with the weights plus the cache, the VAE tiles first. On a cache +smaller than the default budgets the generator shrinks a stage's budget to +the room the prefix leaves (`[AR] Frame budget clamped to N by the KV +cache`), the stream ends there marked truncated; it refuses only when that +room is below the stage's `min_tokens`. The cost of STRICT is one reload of each half per song from the page cache, about a second on the pod, `--keep-loaded` removes it. @@ -838,7 +840,8 @@ GET /job?id=N Poll job status GET /job?id=N&result=1 Fetch job result multipart/mixed, boundary yue2-batch-boundary: per track, song-major, one application/json replay request part (the request carrying the - semantic stream, the score and the seeds of that track), one + semantic stream, the score and the seeds of that track, plus + "truncated": true when a stage hit its budget before its end token), one audio/mpeg or audio/wav part and one application/octet-stream part with the latents the audio was decoded from (raw f32 [T, 64], the .vae layout); for a VAE job, application/octet-stream latents from an encode diff --git a/src/generate.h b/src/generate.h index 120b9e1..d904b60 100644 --- a/src/generate.h +++ b/src/generate.h @@ -16,6 +16,7 @@ #include "sampling.h" #include "timer.h" +#include #include #include #include @@ -61,7 +62,7 @@ static bool yue2_generate(Qwen3LM * lm, const std::vector> & prefixes, const std::vector> & negatives, float cfg_scale, - const Yue2Sampling & s, + const Yue2Sampling & requested, int64_t seed, Yue2Phase phase, std::vector * out, @@ -74,14 +75,24 @@ static bool yue2_generate(Qwen3LM * lm, fprintf(stderr, "[AR] FATAL: guidance %.3f needs an unconditional prefix per sequence\n", (double) cfg_scale); return false; } - // Every set holds its prefix, the budget and the end token + // Every set holds its prefix, the budget and the end token. A budget the + // context cannot hold shrinks to the room left, and the stream ends + // truncated there; only a room below the stage's minimum stops it. + Yue2Sampling s = requested; for (int i = 0; i < B; i++) { size_t longest = guided && negatives[i].size() > prefixes[i].size() ? negatives[i].size() : prefixes[i].size(); - if ((int) longest + s.max_tokens + 1 > context) { - fprintf(stderr, "[AR] FATAL: prefix %zu + budget %d + end exceeds context %d\n", longest, s.max_tokens, - context); + int room = context - (int) longest - 1; + if (room >= s.max_tokens) { + continue; + } + if (room < std::max(1, s.min_tokens)) { + fprintf(stderr, "[AR] FATAL: prefix %zu leaves %d of context %d, the stage needs at least %d\n", longest, + room > 0 ? room : 0, context, std::max(1, s.min_tokens)); return false; } + fprintf(stderr, "[AR] %s budget clamped to %d by the KV cache (prompt %zu tokens)\n", + phase == YUE2_PHASE_ABC ? "Score" : "Frame", room, longest); + s.max_tokens = room; } const int N = guided ? 2 * B : B; diff --git a/tools/yue-server.cpp b/tools/yue-server.cpp index 56ed439..c46803b 100644 --- a/tools/yue-server.cpp +++ b/tools/yue-server.cpp @@ -452,6 +452,26 @@ static std::string json_string(const char * key, const std::string & value) { return out; } +// The JSON part of a track: its replay request, and "truncated" when a stage +// hit its budget before the end token. A resubmit ignores the extra key. +static std::string track_json(const Yue2Request & replay, bool truncated) { + std::string json = request_to_json(&replay); + if (!truncated) { + return json; + } + yyjson_doc * read = yyjson_read(json.c_str(), json.size(), 0); + yyjson_mut_doc * doc = yyjson_doc_mut_copy(read, NULL); + yyjson_doc_free(read); + yyjson_mut_obj_add_bool(doc, yyjson_mut_doc_get_root(doc), "truncated", true); + char * text = yyjson_mut_write(doc, 0, NULL); + std::string out = text ? text : json; + if (text) { + free(text); + } + yyjson_mut_doc_free(doc); + return out; +} + static void handle_props(const httplib::Request &, httplib::Response & res) { Yue2Request d; request_init(&d); @@ -686,7 +706,7 @@ static void run_job(std::shared_ptr job, Yue2Request request) { } Yue2Request replay = request_replay(request, song.score.empty() ? request.abc : song.score, pipeline_format_tokens(song.tokens), (int) t / M, (int) t % M); - request_parts.push_back(request_to_json(&replay)); + request_parts.push_back(track_json(replay, song.truncated)); latents.push_back(std::move(song.latents)); }