diff --git a/README.md b/README.md index 7e8cac6..017b99d 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,9 @@ result as multipart/mixed, one pair per track: a JSON replay request part (the request carrying the semantic stream, the score and the seeds of the track) then the audio part (MP3 or WAV, selected by `output_format` in the request). `lm_batch_size` songs times `synth_batch_size` noise variations -come out song-major. +come out song-major. Each song is decoded as soon as its latents stand: +while the job runs, the status names `tracks_ready`, and +**GET /job?id=N&result=1&track=K** fetches track K alone in the same form. **POST /job?id=N&cancel=1** cancels a running job. **GET /health** - Returns `{"status":"ok"}`. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2a35aac..9f788ac 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -833,7 +833,13 @@ POST /vae Submit a VAE job, returns job ID latents that are not whole frames within the context GET /job?id=N Poll job status - response: {"status":"running|done|failed|cancelled"} + response: {"status":"running|done|failed|cancelled","tracks_ready":K} + tracks_ready: the tracks of a synth job decoded so far, song-major + +GET /job?id=N&result=1&track=K Fetch track K of a job still running + the multipart of the whole result with that track alone + 404 while track K is not ready, and once the job is done (the whole + result holds every track then) GET /job?id=N&result=1 Fetch job result multipart/mixed, boundary yue2-batch-boundary: per track, song-major, diff --git a/src/pipeline.h b/src/pipeline.h index f5a2159..d69f285 100644 --- a/src/pipeline.h +++ b/src/pipeline.h @@ -416,11 +416,15 @@ static bool pipeline_continue_codes(Yue2Pipeline * p, // of a song solve in one NAR graph over the set the AR left complete. With // source_tokens, the song opens on the section of the source the request // names and the AR writes on from it. +// `track_ready` is told of each track as soon as its audio stands, song by song, while the songs after it are still +// being made; it may take the track's audio and latents for itself. static bool pipeline_generate(Yue2Pipeline * p, const Yue2Request & r, std::vector * songs, bool (*cancelled)(void *) = nullptr, - void * cancel_data = nullptr) { + void * cancel_data = nullptr, + void (*track_ready)(void *, size_t, Yue2Song &) = nullptr, + void * track_data = nullptr) { Timer total_timer; Yue2Cot cot; if (!yue2_cot_parse(r.cot, &cot)) { @@ -614,6 +618,7 @@ static bool pipeline_generate(Yue2Pipeline * p, if (!qw3lm_kv_sets(&p->kv, 1)) { return false; } + float seconds = 0.0f; for (int i = 0; i < B; i++) { const int prefix_len = (int) prefixes[i].size(); const int chunk_size = chunk_sizes[i]; @@ -681,36 +686,40 @@ static bool pipeline_generate(Yue2Pipeline * p, fprintf(stderr, "[NAR] Song %d chunk %d/%d: %d frames, cache %d rows, %d forwarded, %.1f s\n", i, start / chunk_size + 1, chunks, frames, ar_len, ar_len - kept, chunk_timer.ms() / 1000.0); } - } - nar_hold.reset(); - VAEGGML * vae = require_vae(p); - if (!vae) { - return false; - } - ModelHandle vae_hold(p->store, vae); - for (size_t t = 0; t < songs->size(); t++) { - Yue2Song & song = (*songs)[t]; - fprintf(stderr, "[VAE] Track %zu/%zu: song %zu variation %zu\n", t + 1, songs->size(), t / M, t % M); - if (!pipeline_decode(p, vae, song.latents.data(), song.T_lat, &song.audio, &song.T_audio, cancelled, - cancel_data)) { + // A song is decoded as soon as its latents stand, so the first of several is heard while the rest are made. + // The NAR lets go first: where the store keeps one group on the card, the VAE takes its place. + nar_hold.reset(); + nar = nullptr; + VAEGGML * vae = require_vae(p); + if (!vae) { return false; } - if (t == 0 && p->dumper.enabled) { - // Interleaved [T_audio, 2] like the torch reference dump - std::vector interleaved((size_t) 2 * song.T_audio); - for (int k = 0; k < song.T_audio; k++) { - interleaved[(size_t) 2 * k] = song.audio[(size_t) k]; - interleaved[(size_t) 2 * k + 1] = song.audio[(size_t) song.T_audio + k]; + ModelHandle vae_hold(p->store, vae); + for (int j = 0; j < M; j++) { + const size_t t = (size_t) i * M + j; + Yue2Song & song = (*songs)[t]; + fprintf(stderr, "[VAE] Track %zu/%zu: song %d variation %d\n", t + 1, songs->size(), i, j); + if (!pipeline_decode(p, vae, song.latents.data(), song.T_lat, &song.audio, &song.T_audio, cancelled, + cancel_data)) { + return false; + } + if (t == 0 && p->dumper.enabled) { + // Interleaved [T_audio, 2] like the torch reference dump + std::vector interleaved((size_t) 2 * song.T_audio); + for (int k = 0; k < song.T_audio; k++) { + interleaved[(size_t) 2 * k] = song.audio[(size_t) k]; + interleaved[(size_t) 2 * k + 1] = song.audio[(size_t) song.T_audio + k]; + } + debug_dump_2d(&p->dumper, "vae_audio", interleaved.data(), song.T_audio, 2); + } + seconds += (float) song.T_audio / (float) YUE2_SAMPLE_RATE; + if (track_ready) { + track_ready(track_data, t, song); } - debug_dump_2d(&p->dumper, "vae_audio", interleaved.data(), song.T_audio, 2); } } - float seconds = 0.0f; - for (size_t t = 0; t < songs->size(); t++) { - seconds += (float) (*songs)[t].T_audio / (float) YUE2_SAMPLE_RATE; - } fprintf(stderr, "[Pipeline] Done: %zu tracks, %.1f s of audio in %.1f s (%.1fx realtime)\n", songs->size(), seconds, total_timer.ms() / 1000.0, seconds / (float) (total_timer.ms() / 1000.0)); return true; diff --git a/tools/yue-server.cpp b/tools/yue-server.cpp index 56ed439..04d32ea 100644 --- a/tools/yue-server.cpp +++ b/tools/yue-server.cpp @@ -170,6 +170,13 @@ enum class JobStatus : int { CANCELLED = 3, }; +// What a finished track is sent as: its replay request, its encoded audio and its latents +struct JobTrack { + std::string request; + std::string audio; + std::vector latents; +}; + struct Job { std::string id; std::atomic status{ JobStatus::RUNNING }; @@ -177,6 +184,13 @@ struct Job { std::string result_mime; std::atomic cancel{ false }; + // The tracks finished so far, in order, while the job still runs: GET /job names how many there are and + // &track=N fetches one. Guarded by mtx_tracks. + std::mutex mtx_tracks; + std::vector tracks; + const char * tracks_mime = "audio/wav"; + bool tracks_failed = false; + // memory ordering contract: result_body and result_mime are written // before status is stored (seq_cst). the client loads status (seq_cst) // and only reads result fields after seeing done/failed. this guarantees @@ -652,48 +666,75 @@ static void run_vae_decode(std::shared_ptr job, Yue2Request request, std::v job->status.store(JobStatus::DONE); } +// What the pipeline's track_ready callback needs to turn a finished track into the parts the job sends +struct JobTracksSink { + Job * job; + const Yue2Request * request; +}; + +// One part triple per track, song-major. The replay request of a track +// carries its semantic stream, its score and the seeds it consumed, so a +// resubmit reproduces it without the autoregression. +static void job_track_ready(void * data, size_t t, Yue2Song & song) { + auto * sink = (JobTracksSink *) data; + const Yue2Request & request = *sink->request; + const int M = request.synth_batch_size; + JobTrack track; + track.audio = encode_audio(request, song.audio, song.T_audio); + Yue2Request replay = request_replay(request, song.score.empty() ? request.abc : song.score, + pipeline_format_tokens(song.tokens), (int) t / M, (int) t % M); + track.request = request_to_json(&replay); + track.latents = std::move(song.latents); + std::lock_guard lock(sink->job->mtx_tracks); + if (track.audio.empty()) { + sink->job->tracks_failed = true; + return; + } + sink->job->tracks.push_back(std::move(track)); + fprintf(stderr, "[Server] Job %s: track %zu ready\n", sink->job->id.c_str(), t + 1); +} + static void run_job(std::shared_ptr job, Yue2Request request) { active_job_set(job); fprintf(stderr, "[Server] Job %s: %s\n", job->id.c_str(), request_to_json(&request).c_str()); + bool is_mp3 = false; + WavFormat wav_fmt = WAV_S16; + audio_parse_format(request.output_format.c_str(), is_mp3, wav_fmt); + job->tracks_mime = is_mp3 ? "audio/mpeg" : "audio/wav"; + std::vector songs; - bool ok = pipeline_generate(&g_pipeline, request, &songs, server_cancel_job, (void *) &job->cancel); + JobTracksSink sink = { job.get(), &request }; + bool ok = pipeline_generate(&g_pipeline, request, &songs, server_cancel_job, (void *) &job->cancel, job_track_ready, + &sink); + active_job_set(nullptr); if (!ok) { - active_job_set(nullptr); job->status.store(job->cancel.load() ? JobStatus::CANCELLED : JobStatus::FAILED); return; } - bool is_mp3 = false; - WavFormat wav_fmt = WAV_S16; - audio_parse_format(request.output_format.c_str(), is_mp3, wav_fmt); - - // One part triple per track, song-major. The replay request of a track - // carries its semantic stream, its score and the seeds it consumed, so a - // resubmit reproduces it without the autoregression. - const int M = request.synth_batch_size; std::vector audio_parts; std::vector request_parts; std::vector> latents; - for (size_t t = 0; t < songs.size(); t++) { - Yue2Song & song = songs[t]; - audio_parts.push_back(encode_audio(request, song.audio, song.T_audio)); - if (audio_parts.back().empty()) { - active_job_set(nullptr); + { + std::lock_guard lock(job->mtx_tracks); + if (job->tracks_failed || job->tracks.size() != songs.size()) { job->status.store(JobStatus::FAILED); return; } - Yue2Request replay = request_replay(request, song.score.empty() ? request.abc : song.score, - pipeline_format_tokens(song.tokens), (int) t / M, (int) t % M); - request_parts.push_back(request_to_json(&replay)); - latents.push_back(std::move(song.latents)); + for (const JobTrack & track : job->tracks) { + audio_parts.push_back(track.audio); + request_parts.push_back(track.request); + latents.push_back(track.latents); + } } - - active_job_set(nullptr); - job->result_body = multipart_build_tracks(request_parts, audio_parts, is_mp3 ? "audio/mpeg" : "audio/wav", latents); + job->result_body = multipart_build_tracks(request_parts, audio_parts, job->tracks_mime, latents); job->result_mime = MULTIPART_MIME; job->status.store(JobStatus::DONE); + // the whole result holds every track now: the single ones are not kept twice + std::lock_guard lock(job->mtx_tracks); + job->tracks.clear(); } static void print_usage(const char * prog) { @@ -956,7 +997,28 @@ int main(int argc, char ** argv) { } JobStatus status = job->status.load(); if (!req.has_param("result")) { - res.set_content(json_string("status", job_status_str(status)), "application/json"); + size_t ready = 0; + { + std::lock_guard lock(job->mtx_tracks); + ready = job->tracks.size(); + } + char body[96]; + snprintf(body, sizeof(body), "{\"status\":\"%s\",\"tracks_ready\":%zu}", job_status_str(status), ready); + res.set_content(body, "application/json"); + return; + } + // one finished track of a job still running: the same multipart as the whole result, with that track alone + if (req.has_param("track")) { + const size_t index = (size_t) strtoull(req.get_param_value("track").c_str(), nullptr, 10); + std::lock_guard lock(job->mtx_tracks); + if (index >= job->tracks.size()) { + res.status = 404; + res.set_content(json_string("error", "track not ready"), "application/json"); + return; + } + const JobTrack & track = job->tracks[index]; + res.set_content(multipart_build_tracks({ track.request }, { track.audio }, job->tracks_mime, { track.latents }), + MULTIPART_MIME.c_str()); return; } if (status != JobStatus::DONE) {