Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,9 @@ result as multipart/mixed, one pair per track: a JSON replay request part
(the request carrying the semantic stream, the score and the seeds of the
track) then the audio part (MP3 or WAV, selected by `output_format` in the
request). `lm_batch_size` songs times `synth_batch_size` noise variations
come out song-major.
come out song-major. Each song is decoded as soon as its latents stand:
while the job runs, the status names `tracks_ready`, and
**GET /job?id=N&result=1&track=K** fetches track K alone in the same form.
**POST /job?id=N&cancel=1** cancels a running job.

**GET /health** - Returns `{"status":"ok"}`.
Expand Down
8 changes: 7 additions & 1 deletion docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -833,7 +833,13 @@ POST /vae Submit a VAE job, returns job ID
latents that are not whole frames within the context

GET /job?id=N Poll job status
response: {"status":"running|done|failed|cancelled"}
response: {"status":"running|done|failed|cancelled","tracks_ready":K}
tracks_ready: the tracks of a synth job decoded so far, song-major

GET /job?id=N&result=1&track=K Fetch track K of a job still running
the multipart of the whole result with that track alone
404 while track K is not ready, and once the job is done (the whole
result holds every track then)

GET /job?id=N&result=1 Fetch job result
multipart/mixed, boundary yue2-batch-boundary: per track, song-major,
Expand Down
57 changes: 33 additions & 24 deletions src/pipeline.h
Original file line number Diff line number Diff line change
Expand Up @@ -416,11 +416,15 @@ static bool pipeline_continue_codes(Yue2Pipeline * p,
// of a song solve in one NAR graph over the set the AR left complete. With
// source_tokens, the song opens on the section of the source the request
// names and the AR writes on from it.
// `track_ready` is told of each track as soon as its audio stands, song by song, while the songs after it are still
// being made; it may take the track's audio and latents for itself.
static bool pipeline_generate(Yue2Pipeline * p,
const Yue2Request & r,
std::vector<Yue2Song> * songs,
bool (*cancelled)(void *) = nullptr,
void * cancel_data = nullptr) {
void * cancel_data = nullptr,
void (*track_ready)(void *, size_t, Yue2Song &) = nullptr,
void * track_data = nullptr) {
Timer total_timer;
Yue2Cot cot;
if (!yue2_cot_parse(r.cot, &cot)) {
Expand Down Expand Up @@ -614,6 +618,7 @@ static bool pipeline_generate(Yue2Pipeline * p,
if (!qw3lm_kv_sets(&p->kv, 1)) {
return false;
}
float seconds = 0.0f;
for (int i = 0; i < B; i++) {
const int prefix_len = (int) prefixes[i].size();
const int chunk_size = chunk_sizes[i];
Expand Down Expand Up @@ -681,36 +686,40 @@ static bool pipeline_generate(Yue2Pipeline * p,
fprintf(stderr, "[NAR] Song %d chunk %d/%d: %d frames, cache %d rows, %d forwarded, %.1f s\n", i,
start / chunk_size + 1, chunks, frames, ar_len, ar_len - kept, chunk_timer.ms() / 1000.0);
}
}

nar_hold.reset();
VAEGGML * vae = require_vae(p);
if (!vae) {
return false;
}
ModelHandle vae_hold(p->store, vae);
for (size_t t = 0; t < songs->size(); t++) {
Yue2Song & song = (*songs)[t];
fprintf(stderr, "[VAE] Track %zu/%zu: song %zu variation %zu\n", t + 1, songs->size(), t / M, t % M);
if (!pipeline_decode(p, vae, song.latents.data(), song.T_lat, &song.audio, &song.T_audio, cancelled,
cancel_data)) {
// A song is decoded as soon as its latents stand, so the first of several is heard while the rest are made.
// The NAR lets go first: where the store keeps one group on the card, the VAE takes its place.
nar_hold.reset();
nar = nullptr;
VAEGGML * vae = require_vae(p);
if (!vae) {
return false;
}
if (t == 0 && p->dumper.enabled) {
// Interleaved [T_audio, 2] like the torch reference dump
std::vector<float> interleaved((size_t) 2 * song.T_audio);
for (int k = 0; k < song.T_audio; k++) {
interleaved[(size_t) 2 * k] = song.audio[(size_t) k];
interleaved[(size_t) 2 * k + 1] = song.audio[(size_t) song.T_audio + k];
ModelHandle vae_hold(p->store, vae);
for (int j = 0; j < M; j++) {
const size_t t = (size_t) i * M + j;
Yue2Song & song = (*songs)[t];
fprintf(stderr, "[VAE] Track %zu/%zu: song %d variation %d\n", t + 1, songs->size(), i, j);
if (!pipeline_decode(p, vae, song.latents.data(), song.T_lat, &song.audio, &song.T_audio, cancelled,
cancel_data)) {
return false;
}
if (t == 0 && p->dumper.enabled) {
// Interleaved [T_audio, 2] like the torch reference dump
std::vector<float> interleaved((size_t) 2 * song.T_audio);
for (int k = 0; k < song.T_audio; k++) {
interleaved[(size_t) 2 * k] = song.audio[(size_t) k];
interleaved[(size_t) 2 * k + 1] = song.audio[(size_t) song.T_audio + k];
}
debug_dump_2d(&p->dumper, "vae_audio", interleaved.data(), song.T_audio, 2);
}
seconds += (float) song.T_audio / (float) YUE2_SAMPLE_RATE;
if (track_ready) {
track_ready(track_data, t, song);
}
debug_dump_2d(&p->dumper, "vae_audio", interleaved.data(), song.T_audio, 2);
}
}

float seconds = 0.0f;
for (size_t t = 0; t < songs->size(); t++) {
seconds += (float) (*songs)[t].T_audio / (float) YUE2_SAMPLE_RATE;
}
fprintf(stderr, "[Pipeline] Done: %zu tracks, %.1f s of audio in %.1f s (%.1fx realtime)\n", songs->size(), seconds,
total_timer.ms() / 1000.0, seconds / (float) (total_timer.ms() / 1000.0));
return true;
Expand Down
108 changes: 85 additions & 23 deletions tools/yue-server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -170,13 +170,27 @@ enum class JobStatus : int {
CANCELLED = 3,
};

// What a finished track is sent as: its replay request, its encoded audio and its latents
struct JobTrack {
std::string request;
std::string audio;
std::vector<float> latents;
};

struct Job {
std::string id;
std::atomic<JobStatus> status{ JobStatus::RUNNING };
std::string result_body;
std::string result_mime;
std::atomic<bool> cancel{ false };

// The tracks finished so far, in order, while the job still runs: GET /job names how many there are and
// &track=N fetches one. Guarded by mtx_tracks.
std::mutex mtx_tracks;
std::vector<JobTrack> tracks;
const char * tracks_mime = "audio/wav";
bool tracks_failed = false;

// memory ordering contract: result_body and result_mime are written
// before status is stored (seq_cst). the client loads status (seq_cst)
// and only reads result fields after seeing done/failed. this guarantees
Expand Down Expand Up @@ -652,48 +666,75 @@ static void run_vae_decode(std::shared_ptr<Job> job, Yue2Request request, std::v
job->status.store(JobStatus::DONE);
}

// What the pipeline's track_ready callback needs to turn a finished track into the parts the job sends
struct JobTracksSink {
Job * job;
const Yue2Request * request;
};

// One part triple per track, song-major. The replay request of a track
// carries its semantic stream, its score and the seeds it consumed, so a
// resubmit reproduces it without the autoregression.
static void job_track_ready(void * data, size_t t, Yue2Song & song) {
auto * sink = (JobTracksSink *) data;
const Yue2Request & request = *sink->request;
const int M = request.synth_batch_size;
JobTrack track;
track.audio = encode_audio(request, song.audio, song.T_audio);
Yue2Request replay = request_replay(request, song.score.empty() ? request.abc : song.score,
pipeline_format_tokens(song.tokens), (int) t / M, (int) t % M);
track.request = request_to_json(&replay);
track.latents = std::move(song.latents);
std::lock_guard<std::mutex> lock(sink->job->mtx_tracks);
if (track.audio.empty()) {
sink->job->tracks_failed = true;
return;
}
sink->job->tracks.push_back(std::move(track));
fprintf(stderr, "[Server] Job %s: track %zu ready\n", sink->job->id.c_str(), t + 1);
}

static void run_job(std::shared_ptr<Job> job, Yue2Request request) {
active_job_set(job);
fprintf(stderr, "[Server] Job %s: %s\n", job->id.c_str(), request_to_json(&request).c_str());

bool is_mp3 = false;
WavFormat wav_fmt = WAV_S16;
audio_parse_format(request.output_format.c_str(), is_mp3, wav_fmt);
job->tracks_mime = is_mp3 ? "audio/mpeg" : "audio/wav";

std::vector<Yue2Song> songs;
bool ok = pipeline_generate(&g_pipeline, request, &songs, server_cancel_job, (void *) &job->cancel);
JobTracksSink sink = { job.get(), &request };
bool ok = pipeline_generate(&g_pipeline, request, &songs, server_cancel_job, (void *) &job->cancel, job_track_ready,
&sink);

active_job_set(nullptr);
if (!ok) {
active_job_set(nullptr);
job->status.store(job->cancel.load() ? JobStatus::CANCELLED : JobStatus::FAILED);
return;
}

bool is_mp3 = false;
WavFormat wav_fmt = WAV_S16;
audio_parse_format(request.output_format.c_str(), is_mp3, wav_fmt);

// One part triple per track, song-major. The replay request of a track
// carries its semantic stream, its score and the seeds it consumed, so a
// resubmit reproduces it without the autoregression.
const int M = request.synth_batch_size;
std::vector<std::string> audio_parts;
std::vector<std::string> request_parts;
std::vector<std::vector<float>> latents;
for (size_t t = 0; t < songs.size(); t++) {
Yue2Song & song = songs[t];
audio_parts.push_back(encode_audio(request, song.audio, song.T_audio));
if (audio_parts.back().empty()) {
active_job_set(nullptr);
{
std::lock_guard<std::mutex> lock(job->mtx_tracks);
if (job->tracks_failed || job->tracks.size() != songs.size()) {
job->status.store(JobStatus::FAILED);
return;
}
Yue2Request replay = request_replay(request, song.score.empty() ? request.abc : song.score,
pipeline_format_tokens(song.tokens), (int) t / M, (int) t % M);
request_parts.push_back(request_to_json(&replay));
latents.push_back(std::move(song.latents));
for (const JobTrack & track : job->tracks) {
audio_parts.push_back(track.audio);
request_parts.push_back(track.request);
latents.push_back(track.latents);
}
}

active_job_set(nullptr);
job->result_body = multipart_build_tracks(request_parts, audio_parts, is_mp3 ? "audio/mpeg" : "audio/wav", latents);
job->result_body = multipart_build_tracks(request_parts, audio_parts, job->tracks_mime, latents);
job->result_mime = MULTIPART_MIME;
job->status.store(JobStatus::DONE);
// the whole result holds every track now: the single ones are not kept twice
std::lock_guard<std::mutex> lock(job->mtx_tracks);
job->tracks.clear();
}

static void print_usage(const char * prog) {
Expand Down Expand Up @@ -956,7 +997,28 @@ int main(int argc, char ** argv) {
}
JobStatus status = job->status.load();
if (!req.has_param("result")) {
res.set_content(json_string("status", job_status_str(status)), "application/json");
size_t ready = 0;
{
std::lock_guard<std::mutex> lock(job->mtx_tracks);
ready = job->tracks.size();
}
char body[96];
snprintf(body, sizeof(body), "{\"status\":\"%s\",\"tracks_ready\":%zu}", job_status_str(status), ready);
res.set_content(body, "application/json");
return;
}
// one finished track of a job still running: the same multipart as the whole result, with that track alone
if (req.has_param("track")) {
const size_t index = (size_t) strtoull(req.get_param_value("track").c_str(), nullptr, 10);
std::lock_guard<std::mutex> lock(job->mtx_tracks);
if (index >= job->tracks.size()) {
res.status = 404;
res.set_content(json_string("error", "track not ready"), "application/json");
return;
}
const JobTrack & track = job->tracks[index];
res.set_content(multipart_build_tracks({ track.request }, { track.audio }, job->tracks_mime, { track.latents }),
MULTIPART_MIME.c_str());
return;
}
if (status != JobStatus::DONE) {
Expand Down