From 902f89187a3d73aacc648cf9f60846ad2a4f4d0b Mon Sep 17 00:00:00 2001 From: Th-Underscore Date: Wed, 2 Sep 2026 23:01:50 -0400 Subject: [PATCH 1/2] BreezeTTS sub-chunk streaming via resumable AR stepper Refactor Impl::generate into begin/step/end_stream (offline path delegates, bit-identical output). Session emits multiple SSE deltas per text chunk via prefix decode with lookahead margin. Opt-in via stream_subchunk/stream_frames_per_event/stream_lookahead_margin; default path unchanged. 40/40 ctest green. --- include/engine/models/breeze_tts/generator.h | 16 + include/engine/models/breeze_tts/session.h | 9 + model_specs/breeze_tts.json | 29 +- src/models/breeze_tts/generator.cpp | 372 ++++++++++++++----- src/models/breeze_tts/session.cpp | 113 ++++++ 5 files changed, 435 insertions(+), 104 deletions(-) diff --git a/include/engine/models/breeze_tts/generator.h b/include/engine/models/breeze_tts/generator.h index 7156ea6b5..2096ae0d9 100644 --- a/include/engine/models/breeze_tts/generator.h +++ b/include/engine/models/breeze_tts/generator.h @@ -29,6 +29,12 @@ struct BreezeGenerationRequest { uint64_t seed = 0; }; +struct BreezeStreamStep { + std::vector new_codes; + int64_t new_frames = 0; + bool done = false; +}; + class BreezeGeneratorRuntime { public: BreezeGeneratorRuntime( @@ -43,6 +49,16 @@ class BreezeGeneratorRuntime { engine::runtime::AudioBuffer generate(const BreezeGenerationRequest & request); BreezeSpeechCodes encode_reference(const engine::runtime::AudioBuffer & audio) const; + // Resumable sub-chunk streaming. begin_stream runs prompts + prefills; + // step_stream runs up to max_new_frames AR frames; end_stream releases + // graphs. Only one stream may be active at a time. decode_codes runs the + // codec over a code prefix without releasing graphs (caller brackets with + // begin/end_stream). + void begin_stream(const BreezeGenerationRequest & request); + BreezeStreamStep step_stream(size_t max_new_frames); + void end_stream(); + engine::runtime::AudioBuffer decode_codes(const std::vector & codes, int64_t frames); + private: struct Impl; std::unique_ptr impl_; diff --git a/include/engine/models/breeze_tts/session.h b/include/engine/models/breeze_tts/session.h index 8c839da0a..9641a907c 100644 --- a/include/engine/models/breeze_tts/session.h +++ b/include/engine/models/breeze_tts/session.h @@ -74,11 +74,20 @@ class BreezeTTSSession final std::unique_ptr generator_; engine::runtime::CacheSlots reference_cache_; std::optional uncached_reference_; + std::optional next_subchunk_event(); std::vector stream_chunk_requests_; std::optional stream_reference_codes_; engine::runtime::AudioBuffer stream_merged_audio_; size_t stream_chunk_index_ = 0; bool stream_started_ = false; + bool stream_subchunk_ = false; + size_t stream_frames_per_event_ = 32; + int64_t stream_lookahead_margin_ = 12; + bool stream_chunk_active_ = false; + std::vector stream_codes_; + int64_t stream_total_frames_ = 0; + size_t stream_emitted_samples_ = 0; + size_t stream_event_seq_ = 0; }; } // namespace engine::models::breeze_tts diff --git a/model_specs/breeze_tts.json b/model_specs/breeze_tts.json index d94e62cbf..28f8e8a01 100644 --- a/model_specs/breeze_tts.json +++ b/model_specs/breeze_tts.json @@ -127,6 +127,29 @@ "required": false, "min": 0, "default": 0 + }, + { + "name": "stream_subchunk", + "type": "bool", + "description": "Emit sub-chunk streaming events (multiple audio deltas per text chunk) instead of one event per chunk.", + "required": false, + "default": false + }, + { + "name": "stream_frames_per_event", + "type": "int", + "description": "Acoustic frames generated per sub-chunk streaming event.", + "required": false, + "min": 1, + "default": 32 + }, + { + "name": "stream_lookahead_margin", + "type": "int", + "description": "Trailing codec frames withheld from emission to absorb convolutional boundary artifacts.", + "required": false, + "min": 0, + "default": 12 } ], "session": [ @@ -167,7 +190,11 @@ "type": "enum", "description": "Attention lowering; auto probes the backend and falls back to eager on GPUs without a flash kernel (e.g. sm70); default auto.", "required": false, - "values": ["auto", "flash", "eager"], + "values": [ + "auto", + "flash", + "eager" + ], "default": "auto" } ], diff --git a/src/models/breeze_tts/generator.cpp b/src/models/breeze_tts/generator.cpp index 54b5e360b..47ee29221 100644 --- a/src/models/breeze_tts/generator.cpp +++ b/src/models/breeze_tts/generator.cpp @@ -1023,136 +1023,284 @@ struct BreezeGeneratorRuntime::Impl { } }); engine::debug::timing_log_scalar("breeze_tts.generate.prompt_ms", prompt_ms); - text_encoder.release_runtime_graphs(); - double backbone_cond_decode_ms = 0.0; - double backbone_uncond_decode_ms = 0.0; +text_encoder.release_runtime_graphs(); + begin_stream_from_prompt( + request, std::move(cond_embeddings), std::move(uncond_embeddings), cond_steps, uncond_steps); + BreezeStreamStep final; + while (!final.done) { + final = step_stream(static_cast(request.max_tokens)); + } + end_stream(); + const auto & codes = stream_codes_cache_; + if (codes.empty()) { + throw std::runtime_error("BreezeTTS generated no audio codes"); + } + if (config.num_codebooks <= 0 || static_cast(codes.size()) % config.num_codebooks != 0) { + throw std::runtime_error("BreezeTTS audio code count must be divisible by num_codebooks"); + } + BreezeSpeechCodes speech_codes; + speech_codes.codes = codes; + speech_codes.code_groups = config.num_codebooks; + speech_codes.frames = static_cast(codes.size()) / config.num_codebooks; + runtime::AudioBuffer audio = speech_decoder->decode(speech_codes); + speech_decoder->release_runtime_graphs(); + for (float & sample : audio.samples) { + sample = std::clamp(sample, -1.0F, 1.0F); + } + return audio; + } + + struct StreamState { + BreezeGenerationRequest request; + modules::QwenCausalPrefillResult cond; + modules::QwenCausalPrefillResult uncond; + sampling::HfSamplerScratch scratch; + std::mt19937 fallback_rng; + uint64_t sample_call_index = 0; + uint64_t offset_blocks = 0; + sampling::HfSamplingOptions first_options; std::vector first_codebook_history; std::vector codes; + int64_t steps_taken = 0; + bool done = false; + bool use_cfg = false; double backbone_cond_prefill_ms = 0.0; double backbone_uncond_prefill_ms = 0.0; - const double ar_ms = engine::debug::measure_ms([&] { - modules::QwenCausalPrefillResult cond; - backbone_cond_prefill_ms = engine::debug::measure_ms([&] { - cond = backbone_cond->prefill_embeddings(cond_embeddings, cond_steps); + double backbone_cond_decode_ms = 0.0; + double backbone_uncond_decode_ms = 0.0; + }; + std::unique_ptr stream_; + std::vector stream_codes_cache_; + + void begin_stream_from_prompt( + const BreezeGenerationRequest & request, + std::vector cond_embeddings, + std::vector uncond_embeddings, + int64_t cond_steps, + int64_t uncond_steps) { + if (stream_ != nullptr) { + throw std::runtime_error("BreezeTTS stream already active"); + } + const auto & config = assets->config; + const bool use_cfg = request.guidance_scale != 1.0F; + auto state = std::make_unique(); + state->request = request; + state->use_cfg = use_cfg; + state->codes.reserve(static_cast(request.max_tokens * config.num_codebooks)); + state->backbone_cond_prefill_ms = engine::debug::measure_ms([&] { + state->cond = backbone_cond->prefill_embeddings(cond_embeddings, cond_steps); + }); + if (use_cfg) { + state->backbone_uncond_prefill_ms = engine::debug::measure_ms([&] { + state->uncond = backbone_uncond->prefill_embeddings(uncond_embeddings, uncond_steps); }); - std::optional uncond; - if (use_cfg) { - uncond.emplace(); - backbone_uncond_prefill_ms = engine::debug::measure_ms([&] { - *uncond = backbone_uncond->prefill_embeddings(uncond_embeddings, uncond_steps); - }); + } + backbone_cond->start_decode_embeddings(state->cond.state, cond_steps + request.max_tokens); + if (use_cfg) { + backbone_uncond->start_decode_embeddings(state->uncond.state, uncond_steps + request.max_tokens); + } + state->scratch.reserve_vocab(static_cast(config.lm_head_size)); + state->fallback_rng = std::mt19937(static_cast(request.seed)); + state->first_options.do_sample = true; + state->first_options.temperature = request.temperature; + state->first_options.top_k = request.top_k; + state->first_options.top_p = request.top_p; + state->first_options.repetition_penalty = kRepetitionPenalty; + state->first_options.min_tokens_to_keep = 1; + stream_ = std::move(state); + } + + void begin_stream(const BreezeGenerationRequest & request) { + if (request.text.empty()) { + throw std::runtime_error("BreezeTTS requires text"); + } + const auto & config = assets->config; + BreezeSpeechCodes reference; + if (request.reference_codes.has_value()) { + reference = *request.reference_codes; + } else if (request.reference_audio.has_value()) { + reference = speech_encoder->encode(*request.reference_audio); + speech_encoder->release_runtime_graphs(); + } + std::vector reference_codes; + int64_t reference_frames = 0; + if (!reference.codes.empty()) { + if (reference.frames < 0 || reference.code_groups <= 0) { + throw std::runtime_error("BreezeTTS speech codes have invalid shape"); } - backbone_cond->start_decode_embeddings(cond.state, cond_steps + request.max_tokens); - if (use_cfg) { - backbone_uncond->start_decode_embeddings(uncond->state, uncond_steps + request.max_tokens); + if (static_cast(reference.codes.size()) != reference.frames * reference.code_groups) { + throw std::runtime_error("BreezeTTS speech code count does not match shape"); } - - sampling::HfSamplerScratch scratch; - scratch.reserve_vocab(static_cast(config.lm_head_size)); - std::mt19937 fallback_rng(static_cast(request.seed)); - uint64_t sample_call_index = 0; - uint64_t offset_blocks = 0; - sampling::HfSamplingOptions first_options; - first_options.do_sample = true; - first_options.temperature = request.temperature; - first_options.top_k = request.top_k; - first_options.top_p = request.top_p; - first_options.repetition_penalty = kRepetitionPenalty; - first_options.min_tokens_to_keep = 1; - - codes.reserve(static_cast(request.max_tokens * config.num_codebooks)); - for (int64_t step = 0; step < request.max_tokens; ++step) { - if (use_cfg && cond.logits.size() != uncond->logits.size()) { - throw std::runtime_error("BreezeTTS CFG logits shape mismatch"); + reference_codes = reference.codes; + reference_frames = static_cast(reference_codes.size()) / config.num_codebooks; + } + BreezePromptBranch cond_branch; + BreezePromptBranch uncond_branch; + std::vector cond_embeddings; + std::vector uncond_embeddings; + int64_t cond_steps = 0; + int64_t uncond_steps = 0; + const bool use_cfg = request.guidance_scale != 1.0F; + const double prompt_ms = engine::debug::measure_ms([&] { + if (!reference_codes.empty()) { + if (request.reference_text.empty()) { + throw std::runtime_error("BreezeTTS clone requires reference_text"); } - std::vector logits; + cond_branch = tokenizer.build_clone(request.text, request.instruction, request.reference_text, reference_frames); if (use_cfg) { - logits.resize(cond.logits.size()); - for (size_t i = 0; i < logits.size(); ++i) { - logits[i] = uncond->logits[i] + request.guidance_scale * (cond.logits[i] - uncond->logits[i]); - } - } else { - logits = cond.logits; - } - suppress_reserved(logits, kCodecCodebookSize, config.vocab_size); - const int32_t first_token = sample_logits( - std::move(logits), - first_codebook_history, - first_options, - scratch, - fallback_rng, - sampling_policy.cuda_fast_path ? &sampling_policy : nullptr, - request.seed, - sample_call_index, - offset_blocks, - "BreezeTTS semantic sampler"); - if (first_token == config.vocab_size) { - break; - } - if (first_token == config.codebook_pad_token_id) { - continue; + uncond_branch = tokenizer.build_clone_negative(request.text, request.reference_text, reference_frames); } - const auto frame = generate_frame( - cond.hidden, - use_cfg ? uncond->hidden : cond.hidden, - first_token, - request, - scratch, - fallback_rng, - sample_call_index, - offset_blocks); - first_codebook_history.push_back(first_token); - codes.insert(codes.end(), frame.begin(), frame.end()); - const auto embedded = frame_embedding( - weights->audio_embedding, - config.num_codebooks * config.vocab_size, - config.hidden_size, - config.vocab_size, - frame); - modules::QwenCausalDecodeStepResult cond_step; - backbone_cond_decode_ms += engine::debug::measure_ms([&] { - cond_step = backbone_cond->decode_embedding(embedded); - }); - cond.logits = cond_step.logits; - cond.hidden = cond_step.hidden; + } else { + cond_branch = tokenizer.build_tts_instruction(request.text, request.instruction); if (use_cfg) { - modules::QwenCausalDecodeStepResult uncond_step; - backbone_uncond_decode_ms += engine::debug::measure_ms([&] { - uncond_step = backbone_uncond->decode_embedding(embedded); - }); - uncond->logits = uncond_step.logits; - uncond->hidden = uncond_step.hidden; + uncond_branch = tokenizer.build_tts_plain(request.text); } } + cond_embeddings = merge_prompt(cond_branch, reference_codes); + cond_steps = static_cast(cond_branch.input_ids.size()); + if (use_cfg) { + uncond_embeddings = merge_prompt(uncond_branch, reference_codes); + uncond_steps = static_cast(uncond_branch.input_ids.size()); + } }); - engine::debug::timing_log_scalar("breeze_tts.ar.total_ms", ar_ms); - engine::debug::timing_log_scalar("breeze_tts.ar.backbone_cond_prefill_ms", backbone_cond_prefill_ms); - engine::debug::timing_log_scalar("breeze_tts.ar.backbone_uncond_prefill_ms", backbone_uncond_prefill_ms); - engine::debug::timing_log_scalar("breeze_tts.ar.backbone_cond_decode_ms", backbone_cond_decode_ms); - engine::debug::timing_log_scalar("breeze_tts.ar.backbone_uncond_decode_ms", backbone_uncond_decode_ms); - backbone_cond->release_runtime_graphs(); - if (use_cfg) { - backbone_uncond->release_runtime_graphs(); + engine::debug::timing_log_scalar("breeze_tts.generate.prompt_ms", prompt_ms); + text_encoder.release_runtime_graphs(); + begin_stream_from_prompt( + request, std::move(cond_embeddings), std::move(uncond_embeddings), cond_steps, uncond_steps); + } + + // Runs one AR step. Returns 0 = frame emitted, 1 = pad skipped, 2 = EOS/budget end. + int step_frame_once() { + const auto & config = assets->config; + StreamState & state = *stream_; + const BreezeGenerationRequest & request = state.request; + if (state.done || state.steps_taken >= request.max_tokens) { + state.done = true; + return 2; } - depth_pair->release_runtime_graphs(); - if (codes.empty()) { - throw std::runtime_error("BreezeTTS generated no audio codes"); + ++state.steps_taken; + if (state.use_cfg && state.cond.logits.size() != state.uncond.logits.size()) { + throw std::runtime_error("BreezeTTS CFG logits shape mismatch"); } - if (config.num_codebooks <= 0 || static_cast(codes.size()) % config.num_codebooks != 0) { - throw std::runtime_error("BreezeTTS audio code count must be divisible by num_codebooks"); + std::vector logits; + if (state.use_cfg) { + logits.resize(state.cond.logits.size()); + for (size_t i = 0; i < logits.size(); ++i) { + logits[i] = state.uncond.logits[i] + request.guidance_scale * (state.cond.logits[i] - state.uncond.logits[i]); + } + } else { + logits = state.cond.logits; + } + suppress_reserved(logits, kCodecCodebookSize, config.vocab_size); + const int32_t first_token = sample_logits( + std::move(logits), + state.first_codebook_history, + state.first_options, + state.scratch, + state.fallback_rng, + sampling_policy.cuda_fast_path ? &sampling_policy : nullptr, + request.seed, + state.sample_call_index, + state.offset_blocks, + "BreezeTTS semantic sampler"); + if (first_token == config.vocab_size) { + state.done = true; + return 2; } + if (first_token == config.codebook_pad_token_id) { + return 1; + } + const auto frame = generate_frame( + state.cond.hidden, + state.use_cfg ? state.uncond.hidden : state.cond.hidden, + first_token, + request, + state.scratch, + state.fallback_rng, + state.sample_call_index, + state.offset_blocks); + state.first_codebook_history.push_back(first_token); + state.codes.insert(state.codes.end(), frame.begin(), frame.end()); + const auto embedded = frame_embedding( + weights->audio_embedding, + config.num_codebooks * config.vocab_size, + config.hidden_size, + config.vocab_size, + frame); + modules::QwenCausalDecodeStepResult cond_step; + state.backbone_cond_decode_ms += engine::debug::measure_ms([&] { + cond_step = backbone_cond->decode_embedding(embedded); + }); + state.cond.logits = cond_step.logits; + state.cond.hidden = cond_step.hidden; + if (state.use_cfg) { + modules::QwenCausalDecodeStepResult uncond_step; + state.backbone_uncond_decode_ms += engine::debug::measure_ms([&] { + uncond_step = backbone_uncond->decode_embedding(embedded); + }); + state.uncond.logits = uncond_step.logits; + state.uncond.hidden = uncond_step.hidden; + } + return 0; + } + + BreezeStreamStep step_stream(size_t max_new_frames) { + if (stream_ == nullptr) { + throw std::runtime_error("BreezeTTS stream not started"); + } + BreezeStreamStep out; + const size_t base = stream_->codes.size(); + while (out.new_frames < static_cast(max_new_frames)) { + const int status = step_frame_once(); + if (status == 0) { + ++out.new_frames; + } else if (status == 2) { + out.done = true; + break; + } + } + if (stream_->done) { + out.done = true; + } + const auto & config = assets->config; + out.new_codes.assign( + stream_->codes.begin() + static_cast(base), stream_->codes.end()); + (void) config; + return out; + } + + void end_stream() { + if (stream_ == nullptr) { + return; + } + engine::debug::timing_log_scalar("breeze_tts.ar.backbone_cond_prefill_ms", stream_->backbone_cond_prefill_ms); + engine::debug::timing_log_scalar("breeze_tts.ar.backbone_uncond_prefill_ms", stream_->backbone_uncond_prefill_ms); + engine::debug::timing_log_scalar("breeze_tts.ar.backbone_cond_decode_ms", stream_->backbone_cond_decode_ms); + engine::debug::timing_log_scalar("breeze_tts.ar.backbone_uncond_decode_ms", stream_->backbone_uncond_decode_ms); + stream_codes_cache_ = std::move(stream_->codes); + stream_.reset(); + backbone_cond->release_runtime_graphs(); + backbone_uncond->release_runtime_graphs(); + depth_pair->release_runtime_graphs(); + } + + runtime::AudioBuffer decode_prefix_codes(const std::vector & codes, int64_t frames) { + const auto & config = assets->config; BreezeSpeechCodes speech_codes; speech_codes.codes = codes; speech_codes.code_groups = config.num_codebooks; - speech_codes.frames = static_cast(codes.size()) / config.num_codebooks; + speech_codes.frames = frames; runtime::AudioBuffer audio = speech_decoder->decode(speech_codes); - speech_decoder->release_runtime_graphs(); for (float & sample : audio.samples) { sample = std::clamp(sample, -1.0F, 1.0F); } return audio; } + void release_decoder_graphs() { + speech_decoder->release_runtime_graphs(); + } + std::shared_ptr assets; core::ExecutionContext & execution; BreezeTextTokenizer tokenizer; @@ -1200,4 +1348,22 @@ BreezeSpeechCodes BreezeGeneratorRuntime::encode_reference(const engine::runtime return impl_->speech_encoder->encode(audio); } +void BreezeGeneratorRuntime::begin_stream(const BreezeGenerationRequest & request) { + impl_->begin_stream(request); +} + +BreezeStreamStep BreezeGeneratorRuntime::step_stream(size_t max_new_frames) { + return impl_->step_stream(max_new_frames); +} + +void BreezeGeneratorRuntime::end_stream() { + impl_->end_stream(); + impl_->release_decoder_graphs(); +} + +engine::runtime::AudioBuffer BreezeGeneratorRuntime::decode_codes( + const std::vector & codes, int64_t frames) { + return impl_->decode_prefix_codes(codes, frames); +} + } // namespace engine::models::breeze_tts diff --git a/src/models/breeze_tts/session.cpp b/src/models/breeze_tts/session.cpp index dd400d5a1..344ab4627 100644 --- a/src/models/breeze_tts/session.cpp +++ b/src/models/breeze_tts/session.cpp @@ -8,6 +8,7 @@ #include "engine/models/breeze_tts/generator.h" #include +#include #include #include #include @@ -287,6 +288,22 @@ void BreezeTTSSession::start_stream(const runtime::TaskRequest & request) { request.voice->speaker->audio.has_value()) { stream_reference_codes_ = resolve_reference_codes(*request.voice->speaker->audio); } + const auto subchunk_opt = runtime::find_option(request.options, {"stream_subchunk"}); + stream_subchunk_ = subchunk_opt.has_value() + ? runtime::parse_bool_option(*subchunk_opt, "stream_subchunk") + : false; + const auto frames_opt = runtime::parse_i64_option(request.options, {"stream_frames_per_event"}); + if (frames_opt.has_value() && *frames_opt > 0) { + stream_frames_per_event_ = static_cast(*frames_opt); + } + const auto margin_opt = runtime::parse_i64_option(request.options, {"stream_lookahead_margin"}); + if (margin_opt.has_value() && *margin_opt >= 0) { + stream_lookahead_margin_ = *margin_opt; + } + engine::debug::trace_log_scalar("breeze_tts.streaming.subchunk", stream_subchunk_ ? 1 : 0); + engine::debug::trace_log_scalar( + "breeze_tts.streaming.frames_per_event", static_cast(stream_frames_per_event_)); + engine::debug::trace_log_scalar("breeze_tts.streaming.lookahead_margin", stream_lookahead_margin_); stream_started_ = true; } @@ -297,6 +314,9 @@ std::optional BreezeTTSSession::next_stream_event() { if (stream_chunk_index_ >= stream_chunk_requests_.size()) { return std::nullopt; } + if (stream_subchunk_) { + return next_subchunk_event(); + } const size_t chunk_index = stream_chunk_index_++; auto chunk_audio = generator_->generate( build_generation_request(stream_chunk_requests_[chunk_index], stream_reference_codes_, chunk_index)); @@ -310,6 +330,90 @@ std::optional BreezeTTSSession::next_stream_event() { return event; } +std::optional BreezeTTSSession::next_subchunk_event() { + size_t guard = 0; + while (true) { + if (++guard > 256) { + throw std::runtime_error("BreezeTTS sub-chunk streaming stalled"); + } + if (stream_chunk_index_ >= stream_chunk_requests_.size()) { + return std::nullopt; + } + const size_t chunk_index = stream_chunk_index_; + if (!stream_chunk_active_) { + generator_->begin_stream(build_generation_request( + stream_chunk_requests_[chunk_index], stream_reference_codes_, chunk_index)); + stream_chunk_active_ = true; + stream_codes_.clear(); + stream_total_frames_ = 0; + stream_emitted_samples_ = 0; + } + BreezeStreamStep step = generator_->step_stream(stream_frames_per_event_); + stream_codes_.insert(stream_codes_.end(), step.new_codes.begin(), step.new_codes.end()); + stream_total_frames_ += step.new_frames; + if (stream_total_frames_ <= 0) { + // EOS before any frame: close the chunk and move on. + generator_->end_stream(); + stream_chunk_active_ = false; + ++stream_chunk_index_; + continue; + } + runtime::AudioBuffer decoded = generator_->decode_codes(stream_codes_, stream_total_frames_); + if (decoded.samples.empty()) { + if (step.done) { + generator_->end_stream(); + stream_chunk_active_ = false; + ++stream_chunk_index_; + continue; + } + continue; + } + const double samples_per_frame = + static_cast(decoded.samples.size()) / static_cast(stream_total_frames_); + size_t emit_end = 0; + if (step.done) { + emit_end = decoded.samples.size(); + } else { + if (stream_total_frames_ <= stream_lookahead_margin_) { + continue; + } + emit_end = static_cast( + std::floor((stream_total_frames_ - stream_lookahead_margin_) * samples_per_frame)); + if (emit_end > decoded.samples.size()) { + emit_end = decoded.samples.size(); + } + if (emit_end <= stream_emitted_samples_) { + continue; + } + } + runtime::AudioBuffer out; + out.sample_rate = decoded.sample_rate; + out.channels = decoded.channels; + out.samples.assign( + decoded.samples.begin() + static_cast(stream_emitted_samples_), + decoded.samples.begin() + static_cast(emit_end)); + stream_emitted_samples_ = emit_end; + runtime::append_audio_buffer(stream_merged_audio_, out); + runtime::StreamEvent event; + const size_t seq = stream_event_seq_++; + event.named_audio_outputs.push_back({ + "chunk_" + std::to_string(chunk_index) + "_part_" + std::to_string(seq), + std::move(out), + {}, + }); + if (step.done) { + generator_->end_stream(); + stream_chunk_active_ = false; + ++stream_chunk_index_; + } + engine::debug::trace_log_scalar( + "breeze_tts.streaming.chunk_frames", static_cast(stream_total_frames_)); + engine::debug::trace_log_scalar( + "breeze_tts.streaming.emitted_samples", static_cast(stream_emitted_samples_)); + return event; + } +} + void BreezeTTSSession::set_stream_event_sink(runtime::StreamEventCallback sink) { (void)sink; } @@ -327,11 +431,20 @@ runtime::TaskResult BreezeTTSSession::finish_stream() { } void BreezeTTSSession::reset() { + if (stream_chunk_active_) { + generator_->end_stream(); + stream_chunk_active_ = false; + } stream_chunk_requests_.clear(); stream_reference_codes_.reset(); stream_merged_audio_ = runtime::AudioBuffer{}; stream_chunk_index_ = 0; stream_started_ = false; + stream_subchunk_ = false; + stream_codes_.clear(); + stream_total_frames_ = 0; + stream_emitted_samples_ = 0; + stream_event_seq_ = 0; } runtime::StreamEvent BreezeTTSSession::process_audio_chunk(const runtime::AudioChunk & chunk) { From 1000c2a01f3ca476a7f5749e8f5d1542be5e2e9d Mon Sep 17 00:00:00 2001 From: Th-Underscore Date: Wed, 2 Sep 2026 23:18:47 -0400 Subject: [PATCH 2/2] breeze_tts: free decoder graph before rebuilding replacement Growing prefix decodes in sub-chunk streaming change chunk_frames every event. Building the replacement first peaks at old+new VRAM and fragments the allocator until cudaMalloc fails mid-stream even at ~90% usage. Reset first; strictly reduces peak with no behavior change. --- model_specs/breeze_tts.json | 6 +----- src/models/breeze_tts/speech_decoder.cpp | 8 ++++++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/model_specs/breeze_tts.json b/model_specs/breeze_tts.json index 28f8e8a01..e2dce79e1 100644 --- a/model_specs/breeze_tts.json +++ b/model_specs/breeze_tts.json @@ -190,11 +190,7 @@ "type": "enum", "description": "Attention lowering; auto probes the backend and falls back to eager on GPUs without a flash kernel (e.g. sm70); default auto.", "required": false, - "values": [ - "auto", - "flash", - "eager" - ], + "values": ["auto", "flash", "eager"], "default": "auto" } ], diff --git a/src/models/breeze_tts/speech_decoder.cpp b/src/models/breeze_tts/speech_decoder.cpp index 1edf0c1c4..059f99f16 100644 --- a/src/models/breeze_tts/speech_decoder.cpp +++ b/src/models/breeze_tts/speech_decoder.cpp @@ -1119,14 +1119,18 @@ runtime::AudioBuffer BreezeSpeechDecoderRuntime::decode(const BreezeSpeechCodes const bool graph_rebuilt = graph == nullptr || !graph->matches(*weights_, chunk_frames, execution_context_->backend(), threads); if (graph_rebuilt) { - auto replacement = std::make_unique( + // Free the old graph BEFORE allocating the replacement. Growing + // prefix decodes (streaming) change chunk_frames every event; building + // the replacement first peaks at old+new and fragments VRAM until + // cudaMalloc fails even though a single graph fits comfortably. + graph.reset(); + graph = std::make_unique( weights_, chunk_frames, *execution_context_, *constants_, graph_arena_bytes_, allow_flash_attention_); - graph = std::move(replacement); } auto decoded = graph->run(chunk.data(), chunk.size()); const int64_t drop = context * kDecodeSamplesPerCode;