Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ Debug:
--max-seq <N> KV cache size (default: model context)
--vae-core <N> VAE tile core frames (default: 512)
--vae-halo <N> VAE tile halo frames (default: 16)
--nar-chunk <N> Acoustic chunk frame cap (default: what the context holds)
--no-fa Disable flash attention
--clamp-fp16 Clamp hidden states to FP16 range
```
Expand Down
8 changes: 7 additions & 1 deletion docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,11 @@ over. Each chunk carries its own sequence, `prefix + its own codes +
MUSIC_END`, so it gets its own AR prefill. The nominal case is one single
chunk, and there the sequence is exactly what the generation left in the
cache: the end token completes it and the prefill never runs. A prefix over
6573 tokens is what pushes a full length song to two chunks.
6573 tokens is what pushes a full length song to two chunks. `--nar-chunk`
caps the chunk below that: without flash attention the attention scores of
a chunk grow with the square of its length, so a smaller chunk is what
keeps a long song inside a card's memory while the song itself keeps the
whole context.

The noise is drawn once for the whole song and each chunk takes its view of
it, again the rule of the reference.
Expand Down Expand Up @@ -748,6 +752,7 @@ Debug:
--max-seq <N> KV cache size (default: model context)
--vae-core <N> VAE tile core frames (default: 512)
--vae-halo <N> VAE tile halo frames (default: 16)
--nar-chunk <N> Acoustic chunk frame cap (default: what the context holds)
--no-fa Disable flash attention
--clamp-fp16 Clamp hidden states to FP16 range
--dump <dir> Dump intermediate tensors
Expand Down Expand Up @@ -789,6 +794,7 @@ Debug:
--max-seq <N> KV cache size (default: model context)
--vae-core <N> VAE tile core frames (default: 512)
--vae-halo <N> VAE tile halo frames (default: 16)
--nar-chunk <N> Acoustic chunk frame cap (default: what the context holds)
--no-fa Disable flash attention
--clamp-fp16 Clamp hidden states to FP16 range
```
Expand Down
4 changes: 4 additions & 0 deletions src/pipeline.h
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ struct Yue2PipelineParams {
bool clamp_fp16 = false; // clamp hidden states on sub-Ampere CUDA
int vae_core = 512; // VAE tile core frames
int vae_halo = 16; // VAE tile halo frames
int nar_chunk = 0; // acoustic chunk frame cap, 0 = what the context holds

const char * dump_dir = nullptr; // probe dumps of the first track for the cossim harness
const char * adapters_dir = nullptr; // the adapters requests can stack, none without it
Expand Down Expand Up @@ -573,6 +574,9 @@ static bool pipeline_generate(Yue2Pipeline * p,
context);
return false;
}
if (p->params.nar_chunk > 0 && chunk_sizes[i] > p->params.nar_chunk) {
chunk_sizes[i] = p->params.nar_chunk;
}
for (int j = 0; j < M; j++) {
Yue2Song & song = (*songs)[(size_t) i * M + j];
song.score = scores[i];
Expand Down
3 changes: 3 additions & 0 deletions tools/yue-server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -720,6 +720,7 @@ static void print_usage(const char * prog) {
" --max-seq <N> KV cache size (default: model context)\n"
" --vae-core <N> VAE tile core frames (default: 512)\n"
" --vae-halo <N> VAE tile halo frames (default: 16)\n"
" --nar-chunk <N> Acoustic chunk frame cap (default: what the context holds)\n"
" --no-fa Disable flash attention\n"
" --clamp-fp16 Clamp hidden states to FP16 range\n",
prog);
Expand Down Expand Up @@ -767,6 +768,8 @@ int main(int argc, char ** argv) {
params.vae_core = atoi(argv[++i]);
} else if (!strcmp(argv[i], "--vae-halo") && !last) {
params.vae_halo = atoi(argv[++i]);
} else if (!strcmp(argv[i], "--nar-chunk") && !last) {
params.nar_chunk = atoi(argv[++i]);
} else if (!strcmp(argv[i], "--no-fa")) {
params.no_fa = true;
} else if (!strcmp(argv[i], "--clamp-fp16")) {
Expand Down
3 changes: 3 additions & 0 deletions tools/yue-synth.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ static void print_usage(const char * prog) {
" --max-seq <N> KV cache size (default: model context)\n"
" --vae-core <N> VAE tile core frames (default: 512)\n"
" --vae-halo <N> VAE tile halo frames (default: 16)\n"
" --nar-chunk <N> Acoustic chunk frame cap (default: what the context holds)\n"
" --no-fa Disable flash attention\n"
" --clamp-fp16 Clamp hidden states to FP16 range\n"
" --dump <dir> Dump intermediate tensors\n",
Expand Down Expand Up @@ -121,6 +122,8 @@ int main(int argc, char ** argv) {
params.vae_core = atoi(argv[++i]);
} else if (!strcmp(argv[i], "--vae-halo") && !last) {
params.vae_halo = atoi(argv[++i]);
} else if (!strcmp(argv[i], "--nar-chunk") && !last) {
params.nar_chunk = atoi(argv[++i]);
} else if (!strcmp(argv[i], "--no-fa")) {
params.no_fa = true;
} else if (!strcmp(argv[i], "--clamp-fp16")) {
Expand Down