From 19cc620a55759e2c28d1fe76b8a8dd9cd5aa7249 Mon Sep 17 00:00:00 2001 From: Nerual Dreming Date: Thu, 12 Nov 2026 14:01:18 +0300 Subject: [PATCH] Add --nar-chunk to cap the acoustic chunk below what the context holds --- README.md | 1 + docs/ARCHITECTURE.md | 8 +++++++- src/pipeline.h | 4 ++++ tools/yue-server.cpp | 3 +++ tools/yue-synth.cpp | 3 +++ 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e8cac6..424202c 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,7 @@ Debug: --max-seq KV cache size (default: model context) --vae-core VAE tile core frames (default: 512) --vae-halo VAE tile halo frames (default: 16) + --nar-chunk Acoustic chunk frame cap (default: what the context holds) --no-fa Disable flash attention --clamp-fp16 Clamp hidden states to FP16 range ``` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2a35aac..3591e0d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -517,7 +517,11 @@ over. Each chunk carries its own sequence, `prefix + its own codes + MUSIC_END`, so it gets its own AR prefill. The nominal case is one single chunk, and there the sequence is exactly what the generation left in the cache: the end token completes it and the prefill never runs. A prefix over -6573 tokens is what pushes a full length song to two chunks. +6573 tokens is what pushes a full length song to two chunks. `--nar-chunk` +caps the chunk below that: without flash attention the attention scores of +a chunk grow with the square of its length, so a smaller chunk is what +keeps a long song inside a card's memory while the song itself keeps the +whole context. The noise is drawn once for the whole song and each chunk takes its view of it, again the rule of the reference. @@ -748,6 +752,7 @@ Debug: --max-seq KV cache size (default: model context) --vae-core VAE tile core frames (default: 512) --vae-halo VAE tile halo frames (default: 16) + --nar-chunk Acoustic chunk frame cap (default: what the context holds) --no-fa Disable flash attention --clamp-fp16 Clamp hidden states to FP16 range --dump Dump intermediate tensors @@ -789,6 +794,7 @@ Debug: --max-seq KV cache size (default: model context) --vae-core VAE tile core frames (default: 512) --vae-halo VAE tile halo frames (default: 16) + --nar-chunk Acoustic chunk frame cap (default: what the context holds) --no-fa Disable flash attention --clamp-fp16 Clamp hidden states to FP16 range ``` diff --git a/src/pipeline.h b/src/pipeline.h index f5a2159..6d284e3 100644 --- a/src/pipeline.h +++ b/src/pipeline.h @@ -48,6 +48,7 @@ struct Yue2PipelineParams { bool clamp_fp16 = false; // clamp hidden states on sub-Ampere CUDA int vae_core = 512; // VAE tile core frames int vae_halo = 16; // VAE tile halo frames + int nar_chunk = 0; // acoustic chunk frame cap, 0 = what the context holds const char * dump_dir = nullptr; // probe dumps of the first track for the cossim harness const char * adapters_dir = nullptr; // the adapters requests can stack, none without it @@ -573,6 +574,9 @@ static bool pipeline_generate(Yue2Pipeline * p, context); return false; } + if (p->params.nar_chunk > 0 && chunk_sizes[i] > p->params.nar_chunk) { + chunk_sizes[i] = p->params.nar_chunk; + } for (int j = 0; j < M; j++) { Yue2Song & song = (*songs)[(size_t) i * M + j]; song.score = scores[i]; diff --git a/tools/yue-server.cpp b/tools/yue-server.cpp index 56ed439..988d002 100644 --- a/tools/yue-server.cpp +++ b/tools/yue-server.cpp @@ -720,6 +720,7 @@ static void print_usage(const char * prog) { " --max-seq KV cache size (default: model context)\n" " --vae-core VAE tile core frames (default: 512)\n" " --vae-halo VAE tile halo frames (default: 16)\n" + " --nar-chunk Acoustic chunk frame cap (default: what the context holds)\n" " --no-fa Disable flash attention\n" " --clamp-fp16 Clamp hidden states to FP16 range\n", prog); @@ -767,6 +768,8 @@ int main(int argc, char ** argv) { params.vae_core = atoi(argv[++i]); } else if (!strcmp(argv[i], "--vae-halo") && !last) { params.vae_halo = atoi(argv[++i]); + } else if (!strcmp(argv[i], "--nar-chunk") && !last) { + params.nar_chunk = atoi(argv[++i]); } else if (!strcmp(argv[i], "--no-fa")) { params.no_fa = true; } else if (!strcmp(argv[i], "--clamp-fp16")) { diff --git a/tools/yue-synth.cpp b/tools/yue-synth.cpp index 27ab67b..e4dcdc2 100644 --- a/tools/yue-synth.cpp +++ b/tools/yue-synth.cpp @@ -58,6 +58,7 @@ static void print_usage(const char * prog) { " --max-seq KV cache size (default: model context)\n" " --vae-core VAE tile core frames (default: 512)\n" " --vae-halo VAE tile halo frames (default: 16)\n" + " --nar-chunk Acoustic chunk frame cap (default: what the context holds)\n" " --no-fa Disable flash attention\n" " --clamp-fp16 Clamp hidden states to FP16 range\n" " --dump Dump intermediate tensors\n", @@ -121,6 +122,8 @@ int main(int argc, char ** argv) { params.vae_core = atoi(argv[++i]); } else if (!strcmp(argv[i], "--vae-halo") && !last) { params.vae_halo = atoi(argv[++i]); + } else if (!strcmp(argv[i], "--nar-chunk") && !last) { + params.nar_chunk = atoi(argv[++i]); } else if (!strcmp(argv[i], "--no-fa")) { params.no_fa = true; } else if (!strcmp(argv[i], "--clamp-fp16")) {