From c074871492f2453cad136503391b84f16c0c7ac0 Mon Sep 17 00:00:00 2001 From: IIIIIllllIIIIIlllll <2432896620@qq.com> Date: Thu, 3 Sep 2026 22:24:33 +0800 Subject: [PATCH 1/4] breeze: chunk the speech-encoder conv stack to bound clone VRAM The encoder graph was built at the exact reference-audio length, so conv activations grew linearly (~45 MiB/s of reference) and every new length triggered a full graph rebuild; a 60 s reference cost ~2.5 GB extra over a 6 s one. Split the encoder into two graphs. The conv stack now runs on fixed 5 s chunks (120000 samples) preceded by a 9600-sample left overlap that covers the stack's exact 5240-sample receptive field; chunk lengths are multiples of the 960x transformer stride, so no per-stage right padding occurs and the discarded overlap frames absorb the zero left pads that represent audio start in the first chunk. Stitched outputs are bit-identical to a single-pass encode of the same input (verified over 68 frames x 16 codebooks). The transformer, downsample, and projections run once over the full frame sequence at frame scale, where even minute-long references cost only tens of MiB. Measured on a 2080 Ti (Vulkan, native q8_0 GGUF, peak minus idle baseline): the VRAM slope over reference length drops from ~45 MiB/s to ~11 MiB/s (remaining slope is the frame-scale transformer graph and the longer AR prefill from reference codes), and a 60 s reference peaks ~1.4 GB lower. Encode time for 60 s improves from 3561 ms to 2197 ms. --- .../engine/models/breeze_tts/speech_encoder.h | 6 +- src/models/breeze_tts/speech_encoder.cpp | 231 ++++++++++++++---- 2 files changed, 189 insertions(+), 48 deletions(-) diff --git a/include/engine/models/breeze_tts/speech_encoder.h b/include/engine/models/breeze_tts/speech_encoder.h index 60da8d924..f2dbae6c4 100644 --- a/include/engine/models/breeze_tts/speech_encoder.h +++ b/include/engine/models/breeze_tts/speech_encoder.h @@ -19,7 +19,8 @@ namespace engine::models { namespace breeze_tts { struct BreezeSpeechEncoderWeights; -class BreezeSpeechEncoderGraph; +class BreezeSpeechEncoderConvGraph; +class BreezeSpeechEncoderTransformerGraph; struct BreezeSpeechEncoderOutput { BreezeSpeechCodes codes; @@ -46,7 +47,8 @@ class BreezeSpeechEncoderRuntime { core::ExecutionContext * execution_context_ = nullptr; size_t graph_arena_bytes_ = 0; std::unique_ptr constants_; - mutable std::unique_ptr graph_; + mutable std::unique_ptr conv_graph_; + mutable std::unique_ptr transformer_graph_; }; } // namespace breeze_tts diff --git a/src/models/breeze_tts/speech_encoder.cpp b/src/models/breeze_tts/speech_encoder.cpp index f4f7d34a5..792397786 100644 --- a/src/models/breeze_tts/speech_encoder.cpp +++ b/src/models/breeze_tts/speech_encoder.cpp @@ -73,6 +73,19 @@ constexpr modules::Conv1dConfig kDownsampleConvConfig{512, 512, 4, 2, 0, 1, fals constexpr modules::Conv1dConfig kSemanticProjectionConfig{512, 256, 1, 1, 0, 1, false}; constexpr modules::Conv1dConfig kAcousticProjectionConfig{512, 256, 1, 1, 0, 1, false}; +// The conv stack downsamples 960x before the transformer (strides 4*5*6*8). +constexpr int64_t kTransformerStride = 960; +// Conv chunks are kChunkSamples of new audio preceded by kChunkOverlapSamples of +// left context. The exact left context the stack needs is the sum of each +// layer's left pad scaled by the cumulative stride: +// 6+2 + 4+2*4 + 5*4+2*20 + 6*20+2*120 + 8*120+2*960 + 2*960 = 5240 samples. +constexpr int64_t kChunkSamples = 120000; // 5 s at 24 kHz, 125 transformer frames +constexpr int64_t kChunkOverlapSamples = 9600; // 10 transformer frames > 5240 +constexpr int64_t kChunkCapacity = kChunkSamples + kChunkOverlapSamples; +static_assert(kChunkSamples % kTransformerStride == 0); +static_assert(kChunkOverlapSamples % kTransformerStride == 0); +constexpr int64_t kChunkFrames = kChunkCapacity / kTransformerStride; + struct GgmlContextDeleter { void operator()(ggml_context * ctx) const noexcept { if (ctx != nullptr) { @@ -415,24 +428,23 @@ std::shared_ptr load_weights( return weights; } -class BreezeSpeechEncoderGraph { +// Conv stack runs on fixed-size chunks (plus left overlap), so its graph +// memory is constant regardless of reference length. Chunk outputs stitch +// exactly: chunk lengths are multiples of kTransformerStride, so no per-stage +// right padding occurs, and discarded overlap frames absorb the zero left +// pads that represent audio start in the first chunk. +class BreezeSpeechEncoderConvGraph { public: - BreezeSpeechEncoderGraph( + BreezeSpeechEncoderConvGraph( std::shared_ptr weights, - int64_t sample_capacity, core::ExecutionContext & execution_context, core::ConstantTensorCache & constants, size_t graph_arena_bytes) : weights_(std::move(weights)), - sample_capacity_(sample_capacity), - frames_((sample_capacity + kDownsampleRate - 1) / kDownsampleRate), backend_(execution_context.backend()), compute_threads_(std::max(1, execution_context.config().threads)) { if (weights_ == nullptr) { - throw std::runtime_error("Breeze speech encoder graph requires weights"); - } - if (sample_capacity_ <= 0) { - throw std::runtime_error("Breeze speech encoder graph requires positive sample capacity"); + throw std::runtime_error("Breeze speech encoder conv graph requires weights"); } if (backend_ == nullptr) { throw std::runtime_error("Breeze speech encoder backend is not initialized"); @@ -445,15 +457,15 @@ class BreezeSpeechEncoderGraph { }; ctx_.reset(ggml_init(params)); if (ctx_ == nullptr) { - throw std::runtime_error("failed to initialize Breeze speech encoder ggml context"); + throw std::runtime_error("failed to initialize Breeze speech encoder conv ggml context"); } core::ModuleBuildContext build_ctx{ ctx_.get(), - "breeze_tts.speech_encoder", + "breeze_tts.speech_encoder.conv", execution_context.backend_type(), }; - auto x = core::make_tensor(build_ctx, GGML_TYPE_F32, core::TensorShape::from_dims({1, 1, sample_capacity_})); + auto x = core::make_tensor(build_ctx, GGML_TYPE_F32, core::TensorShape::from_dims({1, 1, kChunkCapacity})); input_ = x.tensor; constants.begin_graph(); @@ -465,25 +477,123 @@ class BreezeSpeechEncoderGraph { } x = modules::EluModule{}.build(build_ctx, x); x = speech_conv(build_ctx, x, weights_->encoder_convs.back(), kEncoderConvConfigs.back(), kEncoderConvPadModes.back()); - auto seq = modules::TransposeModule({{0, 2, 1, 3}, x.shape.rank}).build(build_ctx, x); seq = core::ensure_backend_addressable_layout(build_ctx, seq); - transformer_frames_ = seq.shape.dims[1]; - positions_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I32, transformer_frames_); - auto positions_value = core::wrap_tensor(positions_, core::TensorShape::from_dims({transformer_frames_}), GGML_TYPE_I32); - attention_mask_ = ggml_new_tensor_4d(ctx_.get(), GGML_TYPE_F16, transformer_frames_, transformer_frames_, 1, 1); + output_ = seq.tensor; + ggml_set_output(output_); + graph_ = ggml_new_graph_custom(ctx_.get(), 32768, false); + ggml_build_forward_expand(graph_, output_); + constants.finish_graph(); + constants.ensure_uploaded(); + + gallocr_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend_)); + if (gallocr_ == nullptr || !ggml_gallocr_alloc_graph(gallocr_, graph_)) { + throw std::runtime_error("failed to allocate Breeze speech encoder conv graph"); + } + } + + ~BreezeSpeechEncoderConvGraph() { + engine::core::release_backend_graph_resources(backend_, graph_); + if (gallocr_ != nullptr) { + ggml_gallocr_free(gallocr_); + } + } + + bool matches(const BreezeSpeechEncoderWeights & weights, ggml_backend_t backend, int threads) const { + return weights_.get() == &weights && backend_ == backend && compute_threads_ == std::max(1, threads); + } + + std::vector run(const std::vector & chunk_input) { + if (static_cast(chunk_input.size()) != kChunkCapacity) { + throw std::runtime_error("Breeze speech encoder conv chunk size mismatch"); + } + ggml_backend_tensor_set(input_, chunk_input.data(), 0, chunk_input.size() * sizeof(float)); + core::set_backend_threads(backend_, compute_threads_); + const ggml_status status = engine::core::compute_backend_graph(backend_, graph_); + ggml_backend_synchronize(backend_); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Breeze speech encoder conv graph compute failed"); + } + std::vector features(static_cast(kHiddenSize * kChunkFrames)); + ggml_backend_tensor_get(output_, features.data(), 0, features.size() * sizeof(float)); + return features; + } + +private: + std::shared_ptr weights_; + std::unique_ptr ctx_; + ggml_tensor * input_ = nullptr; + ggml_tensor * output_ = nullptr; + ggml_cgraph * graph_ = nullptr; + ggml_backend_t backend_ = nullptr; + int compute_threads_ = 1; + ggml_gallocr_t gallocr_ = nullptr; +}; + +// Transformer + downsample + projections run once over the full frame +// sequence; at frame scale (960x downsampled) this graph is a few tens of MiB +// even for minute-long references. Attention is causal, so frames computed +// from right-padded tail regions never affect earlier frames. +class BreezeSpeechEncoderTransformerGraph { +public: + BreezeSpeechEncoderTransformerGraph( + std::shared_ptr weights, + int64_t frames, + core::ExecutionContext & execution_context, + core::ConstantTensorCache & constants, + size_t graph_arena_bytes) + : weights_(std::move(weights)), + frames_(frames), + backend_(execution_context.backend()), + compute_threads_(std::max(1, execution_context.config().threads)) { + if (weights_ == nullptr) { + throw std::runtime_error("Breeze speech encoder transformer graph requires weights"); + } + if (frames_ <= 0) { + throw std::runtime_error("Breeze speech encoder transformer graph requires positive frame count"); + } + if (backend_ == nullptr) { + throw std::runtime_error("Breeze speech encoder backend is not initialized"); + } + + ggml_init_params params{ + /*.mem_size =*/ graph_arena_bytes, + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ctx_.reset(ggml_init(params)); + if (ctx_ == nullptr) { + throw std::runtime_error("failed to initialize Breeze speech encoder transformer ggml context"); + } + + core::ModuleBuildContext build_ctx{ + ctx_.get(), + "breeze_tts.speech_encoder.transformer", + execution_context.backend_type(), + }; + auto seq = core::make_tensor( + build_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, frames_, kHiddenSize})); + input_ = seq.tensor; + + constants.begin_graph(); + positions_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I32, frames_); + auto positions_value = core::wrap_tensor(positions_, core::TensorShape::from_dims({frames_}), GGML_TYPE_I32); + attention_mask_ = ggml_new_tensor_4d(ctx_.get(), GGML_TYPE_F16, frames_, frames_, 1, 1); const auto attention_mask = core::wrap_tensor( attention_mask_, - core::TensorShape::from_dims({1, 1, transformer_frames_, transformer_frames_}), + core::TensorShape::from_dims({1, 1, frames_, frames_}), GGML_TYPE_F16); for (const auto & layer : weights_->transformer_layers) { seq = transformer_block(build_ctx, seq, positions_value, layer, attention_mask); } - x = modules::TransposeModule({{0, 2, 1, 3}, seq.shape.rank}).build(build_ctx, seq); + auto x = modules::TransposeModule({{0, 2, 1, 3}, seq.shape.rank}).build(build_ctx, seq); x = core::ensure_backend_addressable_layout(build_ctx, x); x = speech_conv(build_ctx, x, weights_->downsample, kDownsampleConvConfig, modules::StreamingPadMode::Replicate); auto semantic = speech_conv(build_ctx, x, weights_->semantic_projection, kSemanticProjectionConfig, modules::StreamingPadMode::Constant); auto acoustic = speech_conv(build_ctx, x, weights_->acoustic_projection, kAcousticProjectionConfig, modules::StreamingPadMode::Constant); + output_frames_ = semantic.shape.dims[2]; semantic_output_ = semantic.tensor; acoustic_output_ = acoustic.tensor; ggml_set_output(semantic_output_); @@ -496,55 +606,53 @@ class BreezeSpeechEncoderGraph { gallocr_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend_)); if (gallocr_ == nullptr || !ggml_gallocr_alloc_graph(gallocr_, graph_)) { - throw std::runtime_error("failed to allocate Breeze speech encoder graph"); + throw std::runtime_error("failed to allocate Breeze speech encoder transformer graph"); } - positions_data_.resize(static_cast(transformer_frames_)); - for (int64_t i = 0; i < transformer_frames_; ++i) { + positions_data_.resize(static_cast(frames_)); + for (int64_t i = 0; i < frames_; ++i) { positions_data_[static_cast(i)] = static_cast(i); } if (attention_mask_ != nullptr) { - auto mask = modules::qwen_causal_prefill_mask_values(1, transformer_frames_); + auto mask = modules::qwen_causal_prefill_mask_values(1, frames_); attention_mask_data_ = std::move(mask); } upload_static_inputs(); } - ~BreezeSpeechEncoderGraph() { + ~BreezeSpeechEncoderTransformerGraph() { engine::core::release_backend_graph_resources(backend_, graph_); if (gallocr_ != nullptr) { ggml_gallocr_free(gallocr_); } } - bool matches(const BreezeSpeechEncoderWeights & weights, int64_t samples, ggml_backend_t backend, int threads) const { - return weights_.get() == &weights && sample_capacity_ == samples && backend_ == backend && + bool matches(const BreezeSpeechEncoderWeights & weights, int64_t frames, ggml_backend_t backend, int threads) const { + return weights_.get() == &weights && frames_ == frames && backend_ == backend && compute_threads_ == std::max(1, threads); } - BreezeSpeechEncoderOutput run(const std::vector & waveform) { - if (static_cast(waveform.size()) > sample_capacity_) { - throw std::runtime_error("Breeze speech encoder waveform exceeds graph capacity"); + BreezeSpeechEncoderOutput run(const std::vector & features) { + if (static_cast(features.size()) != kHiddenSize * frames_) { + throw std::runtime_error("Breeze speech encoder transformer input size mismatch"); } upload_static_inputs(); - std::vector padded(static_cast(sample_capacity_), 0.0F); - std::copy(waveform.begin(), waveform.end(), padded.begin()); - ggml_backend_tensor_set(input_, padded.data(), 0, padded.size() * sizeof(float)); + ggml_backend_tensor_set(input_, features.data(), 0, features.size() * sizeof(float)); core::set_backend_threads(backend_, compute_threads_); const ggml_status status = engine::core::compute_backend_graph(backend_, graph_); ggml_backend_synchronize(backend_); if (status != GGML_STATUS_SUCCESS) { - throw std::runtime_error("Breeze speech encoder graph compute failed"); + throw std::runtime_error("Breeze speech encoder transformer graph compute failed"); } BreezeSpeechEncoderOutput out; - out.semantic_projected.resize(static_cast(kQuantizerDim * frames_)); - out.acoustic_projected.resize(static_cast(kQuantizerDim * frames_)); + out.semantic_projected.resize(static_cast(kQuantizerDim * output_frames_)); + out.acoustic_projected.resize(static_cast(kQuantizerDim * output_frames_)); ggml_backend_tensor_get(semantic_output_, out.semantic_projected.data(), 0, out.semantic_projected.size() * sizeof(float)); ggml_backend_tensor_get(acoustic_output_, out.acoustic_projected.data(), 0, out.acoustic_projected.size() * sizeof(float)); return out; } - int64_t frames() const noexcept { - return frames_; + int64_t output_frames() const noexcept { + return output_frames_; } private: @@ -564,9 +672,8 @@ class BreezeSpeechEncoderGraph { } std::shared_ptr weights_; - int64_t sample_capacity_ = 0; int64_t frames_ = 0; - int64_t transformer_frames_ = 0; + int64_t output_frames_ = 0; std::unique_ptr ctx_; ggml_tensor * input_ = nullptr; ggml_tensor * positions_ = nullptr; @@ -623,18 +730,49 @@ BreezeSpeechCodes BreezeSpeechEncoderRuntime::encode(const runtime::AudioBuffer static_cast(kSampleRate)); const int64_t valid_samples = static_cast(waveform.size()); const int64_t frames = std::max(1, (valid_samples + kDownsampleRate - 1) / kDownsampleRate); - const int64_t sample_capacity = valid_samples; + const int64_t transformer_frames = (valid_samples + kTransformerStride - 1) / kTransformerStride; const int threads = std::max(1, execution_context_->config().threads); - if (graph_ == nullptr || !graph_->matches(*weights_, sample_capacity, execution_context_->backend(), threads)) { - graph_.reset(); - graph_ = std::make_unique( + if (conv_graph_ == nullptr || !conv_graph_->matches(*weights_, execution_context_->backend(), threads)) { + conv_graph_.reset(); + conv_graph_ = std::make_unique( weights_, - sample_capacity, *execution_context_, *constants_, graph_arena_bytes_); } - auto out = graph_->run(waveform); + if (transformer_graph_ == nullptr || + !transformer_graph_->matches(*weights_, transformer_frames, execution_context_->backend(), threads)) { + transformer_graph_.reset(); + transformer_graph_ = std::make_unique( + weights_, + transformer_frames, + *execution_context_, + *constants_, + graph_arena_bytes_); + } + + std::vector features(static_cast(kHiddenSize * transformer_frames)); + std::vector chunk_input(static_cast(kChunkCapacity)); + int64_t dst_frame = 0; + for (int64_t pos = 0; pos < valid_samples; pos += kChunkSamples) { + const int64_t overlap = pos > 0 ? kChunkOverlapSamples : 0; + const int64_t fresh = std::min(kChunkSamples, valid_samples - pos); + std::fill(chunk_input.begin(), chunk_input.end(), 0.0F); + std::copy( + waveform.begin() + (pos - overlap), + waveform.begin() + (pos + fresh), + chunk_input.begin()); + const auto chunk_features = conv_graph_->run(chunk_input); + const int64_t skip_frames = overlap / kTransformerStride; + const int64_t keep_frames = (fresh + kTransformerStride - 1) / kTransformerStride; + std::copy_n( + chunk_features.begin() + skip_frames * kHiddenSize, + keep_frames * kHiddenSize, + features.begin() + dst_frame * kHiddenSize); + dst_frame += keep_frames; + } + + auto out = transformer_graph_->run(features); out.codes.frames = frames; out.codes.code_groups = kValidQuantizers; out.codes.codes = quantize_projected(out.semantic_projected, out.acoustic_projected, frames, *weights_); @@ -643,7 +781,8 @@ BreezeSpeechCodes BreezeSpeechEncoderRuntime::encode(const runtime::AudioBuffer } void BreezeSpeechEncoderRuntime::release_runtime_graphs() const { - graph_.reset(); + conv_graph_.reset(); + transformer_graph_.reset(); } } // namespace engine::models::breeze_tts From bbe2712c9cfb733b3584d97587bb2750ce047e3a Mon Sep 17 00:00:00 2001 From: IIIIIllllIIIIIlllll <2432896620@qq.com> Date: Thu, 3 Sep 2026 22:44:04 +0800 Subject: [PATCH 2/4] breeze: bucket speech-encoder transformer graph capacity The transformer graph was rebuilt at the exact frame count for every distinct reference length. Round the capacity up to 125-frame (5 s) buckets so lengths within a bucket share one graph. Unused bucket frames are replicate-padded to match the downsample conv's Replicate right pad; causal attention keeps padding frames invisible to real frames. Verified bit-identical reference codes vs exact-length graphs at 6 s and 15 s; odd lengths show sub-1% last-frame diffs from flash-attention tiling, the same accepted noise class as the pre-existing length sensitivity. Single-run peak VRAM is unchanged. --- src/models/breeze_tts/speech_encoder.cpp | 39 ++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/src/models/breeze_tts/speech_encoder.cpp b/src/models/breeze_tts/speech_encoder.cpp index 792397786..b1d59b400 100644 --- a/src/models/breeze_tts/speech_encoder.cpp +++ b/src/models/breeze_tts/speech_encoder.cpp @@ -85,6 +85,10 @@ constexpr int64_t kChunkCapacity = kChunkSamples + kChunkOverlapSamples; static_assert(kChunkSamples % kTransformerStride == 0); static_assert(kChunkOverlapSamples % kTransformerStride == 0); constexpr int64_t kChunkFrames = kChunkCapacity / kTransformerStride; +// The transformer graph is built at frame capacities rounded up to this +// bucket, so reference lengths within a bucket share one graph instead of +// rebuilding per exact length. +constexpr int64_t kTransformerFrameBucket = kChunkSamples / kTransformerStride; struct GgmlContextDeleter { void operator()(ggml_context * ctx) const noexcept { @@ -731,6 +735,8 @@ BreezeSpeechCodes BreezeSpeechEncoderRuntime::encode(const runtime::AudioBuffer const int64_t valid_samples = static_cast(waveform.size()); const int64_t frames = std::max(1, (valid_samples + kDownsampleRate - 1) / kDownsampleRate); const int64_t transformer_frames = (valid_samples + kTransformerStride - 1) / kTransformerStride; + const int64_t graph_frames = + (transformer_frames + kTransformerFrameBucket - 1) / kTransformerFrameBucket * kTransformerFrameBucket; const int threads = std::max(1, execution_context_->config().threads); if (conv_graph_ == nullptr || !conv_graph_->matches(*weights_, execution_context_->backend(), threads)) { conv_graph_.reset(); @@ -741,17 +747,17 @@ BreezeSpeechCodes BreezeSpeechEncoderRuntime::encode(const runtime::AudioBuffer graph_arena_bytes_); } if (transformer_graph_ == nullptr || - !transformer_graph_->matches(*weights_, transformer_frames, execution_context_->backend(), threads)) { + !transformer_graph_->matches(*weights_, graph_frames, execution_context_->backend(), threads)) { transformer_graph_.reset(); transformer_graph_ = std::make_unique( weights_, - transformer_frames, + graph_frames, *execution_context_, *constants_, graph_arena_bytes_); } - std::vector features(static_cast(kHiddenSize * transformer_frames)); + std::vector features(static_cast(kHiddenSize * graph_frames)); std::vector chunk_input(static_cast(kChunkCapacity)); int64_t dst_frame = 0; for (int64_t pos = 0; pos < valid_samples; pos += kChunkSamples) { @@ -771,8 +777,35 @@ BreezeSpeechCodes BreezeSpeechEncoderRuntime::encode(const runtime::AudioBuffer features.begin() + dst_frame * kHiddenSize); dst_frame += keep_frames; } + // Pad unused bucket frames with the last real frame (not zeros): causal + // attention keeps padding invisible to real frames, and the downsample + // conv's Replicate right pad then sees the same value as an exact-length + // graph would produce. + for (int64_t f = transformer_frames; f < graph_frames; ++f) { + std::copy_n( + features.begin() + (transformer_frames - 1) * kHiddenSize, + kHiddenSize, + features.begin() + f * kHiddenSize); + } auto out = transformer_graph_->run(features); + const int64_t produced_frames = transformer_graph_->output_frames(); + if (produced_frames != frames) { + // Projected outputs are channel-major with stride produced_frames; + // drop the padding frames before quantization. + auto slice_frames = [frames, produced_frames](std::vector & projected) { + std::vector sliced(static_cast(kQuantizerDim * frames)); + for (int64_t dim = 0; dim < kQuantizerDim; ++dim) { + std::copy_n( + projected.begin() + dim * produced_frames, + frames, + sliced.begin() + dim * frames); + } + projected = std::move(sliced); + }; + slice_frames(out.semantic_projected); + slice_frames(out.acoustic_projected); + } out.codes.frames = frames; out.codes.code_groups = kValidQuantizers; out.codes.codes = quantize_projected(out.semantic_projected, out.acoustic_projected, frames, *weights_); From 23eda2ee4030aa18a3c1ab379dd602841ca05144 Mon Sep 17 00:00:00 2001 From: IIIIIllllIIIIIlllll <2432896620@qq.com> Date: Thu, 3 Sep 2026 23:14:34 +0800 Subject: [PATCH 3/4] ggml-vulkan, breeze: fused round-to-bf16 unary op on Vulkan Vulkan previously paid a cast round trip (f32->bf16->f32, two kernels, a bf16 intermediate tensor) at every activation-rounding point of the breeze decoder. Add a round_bf16 compute shader (f32/f16/bf16 in, always f32 out, round-to-nearest-even via the same fp32_to_bf16 bit trick the cpy shaders use), register pipelines indexed by source type, handle the widened f32 dst in the unary pipeline selection and op-support checks, and enable fused_round for Vulkan in the breeze activation-cast policy. Verified bit-identical breeze reference codes vs the cast round trip at 6 s and 15 s references. Peak VRAM on a 2080 Ti drops ~250 MiB at a 60 s reference (5491 -> 5239 MiB); no measurable change at 6 s. --- external/ggml/src/ggml-vulkan/ggml-vulkan.cpp | 23 ++++++++++++++++ .../vulkan-shaders/round_bf16.comp | 26 +++++++++++++++++++ .../vulkan-shaders/vulkan-shaders-gen.cpp | 3 +++ src/models/breeze_tts/generator.cpp | 6 ++--- 4 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 external/ggml/src/ggml-vulkan/vulkan-shaders/round_bf16.comp diff --git a/external/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/external/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 3e9a5217b..f615ae11a 100644 --- a/external/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/external/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -811,6 +811,7 @@ struct vk_device_struct { vk_pipeline pipeline_softplus[2]; vk_pipeline pipeline_step[2]; vk_pipeline pipeline_round[2]; + vk_pipeline pipeline_round_bf16[3]; vk_pipeline pipeline_ceil[2]; vk_pipeline pipeline_floor[2]; vk_pipeline pipeline_trunc[2]; @@ -4750,6 +4751,11 @@ static void ggml_vk_load_shaders(vk_device& device) { CREATE_UNARY(exp) #undef CREATE_UNARY + // round-to-bf16: f32/f16/bf16 in, always f32 out (index by src type). + ggml_vk_create_pipeline(device, device->pipeline_round_bf16[0], "round_bf16_f32", round_bf16_f32_len, round_bf16_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_round_bf16[1], "round_bf16_f16", round_bf16_f16_len, round_bf16_f16_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_round_bf16[2], "round_bf16_bf16", round_bf16_bf16_len, round_bf16_bf16_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f16, "add1_f16_f16", add1_f16_f16_len, add1_f16_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f32, "add1_f16_f32", add1_f16_f32_len, add1_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_add1_f32_f32, "add1_f32_f32", add1_f32_f32_len, add1_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); @@ -9740,6 +9746,18 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const } return nullptr; case GGML_OP_UNARY: + // ROUND_BF16 widens to f32: src may be f32/f16/bf16 while dst is f32. + if (ggml_get_unary_op(dst) == GGML_UNARY_OP_ROUND_BF16) { + if (dst->type != GGML_TYPE_F32) { + return nullptr; + } + switch (src0->type) { + case GGML_TYPE_F32: return ctx->device->pipeline_round_bf16[0]; + case GGML_TYPE_F16: return ctx->device->pipeline_round_bf16[1]; + case GGML_TYPE_BF16: return ctx->device->pipeline_round_bf16[2]; + default: return nullptr; + } + } if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) || (src0->type != dst->type)) { @@ -13516,6 +13534,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr case GGML_UNARY_OP_SOFTPLUS: case GGML_UNARY_OP_STEP: case GGML_UNARY_OP_ROUND: + case GGML_UNARY_OP_ROUND_BF16: case GGML_UNARY_OP_CEIL: case GGML_UNARY_OP_FLOOR: case GGML_UNARY_OP_TRUNC: @@ -15772,6 +15791,10 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && (op->src[0]->type == op->type); + case GGML_UNARY_OP_ROUND_BF16: + return ggml_is_contiguous(op->src[0]) && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) && + (op->type == GGML_TYPE_F32); case GGML_UNARY_OP_SIGMOID: return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && diff --git a/external/ggml/src/ggml-vulkan/vulkan-shaders/round_bf16.comp b/external/ggml/src/ggml-vulkan/vulkan-shaders/round_bf16.comp new file mode 100644 index 000000000..b92937bec --- /dev/null +++ b/external/ggml/src/ggml-vulkan/vulkan-shaders/round_bf16.comp @@ -0,0 +1,26 @@ +#version 450 + +#include "generic_head.glsl" +#include "types.glsl" + +layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; + +void main() { + const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x; + + if (i >= p.KX) { + return; + } + +#if defined(DATA_A_BF16) + const float x = bf16_to_fp32(uint32_t(data_a[i])); +#else + const float x = float(data_a[i]); +#endif + // Round to bf16 precision and widen back to f32, matching the + // f32 -> bf16 -> f32 cast round trip. + data_d[i] = D_TYPE(bf16_to_fp32(fp32_to_bf16(x))); +} diff --git a/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index a533af69f..92fbe3e50 100644 --- a/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -880,6 +880,9 @@ void process_shaders() { string_to_spv("step_f32", "step.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("round_f16", "round.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); string_to_spv("round_f32", "round.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); + string_to_spv("round_bf16_f32", "round_bf16.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); + string_to_spv("round_bf16_f16", "round_bf16.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float"}}); + string_to_spv("round_bf16_bf16", "round_bf16.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "float"}, {"DATA_A_BF16", "1"}}); string_to_spv("ceil_f16", "ceil.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); string_to_spv("ceil_f32", "ceil.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("floor_f16", "floor.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); diff --git a/src/models/breeze_tts/generator.cpp b/src/models/breeze_tts/generator.cpp index 83f149540..28458f122 100644 --- a/src/models/breeze_tts/generator.cpp +++ b/src/models/breeze_tts/generator.cpp @@ -60,9 +60,9 @@ modules::QwenDecoderActivationCastPolicy breeze_bf16_activation_policy(core::Bac } policy.enabled = true; policy.type = GGML_TYPE_BF16; - // CUDA/HIP implement the fused round-to-bf16 unary op; Vulkan does not and - // keeps the cast round trip. - policy.fused_round = backend_type == core::BackendType::Cuda || backend_type == core::BackendType::Hip; + // CUDA/HIP/Vulkan implement the fused round-to-bf16 unary op. + policy.fused_round = backend_type == core::BackendType::Cuda || backend_type == core::BackendType::Hip || + backend_type == core::BackendType::Vulkan; policy.after_input_norm = true; policy.after_qkv_projection = true; policy.after_qk_norm = true; From 3215194efe88d338220b29f2f9675546dc4038ba Mon Sep 17 00:00:00 2001 From: IIIIIllllIIIIIlllll <2432896620@qq.com> Date: Fri, 4 Sep 2026 10:18:33 +0800 Subject: [PATCH 4/4] ggml-vulkan: handle row-strided inputs in fused round-to-bf16 The breeze activation-rounding policy admits row-strided views into ggml_round_bf16 (ggml_is_contiguous_rows gate in qwen_decoder). The Vulkan port dispatched every input to the flat shader, which indexes the source as a contiguous array, so row-strided views read garbage and clone output degenerated into noise. Route non-contiguous inputs to a new round_bf16_strided shader built on generic_unary_head (same pattern as sigmoid_strided), keeping the flat fast path for contiguous inputs. --- external/ggml/src/ggml-vulkan/ggml-vulkan.cpp | 32 ++++++++++++++----- .../vulkan-shaders/round_bf16_strided.comp | 23 +++++++++++++ .../vulkan-shaders/vulkan-shaders-gen.cpp | 5 ++- 3 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 external/ggml/src/ggml-vulkan/vulkan-shaders/round_bf16_strided.comp diff --git a/external/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/external/ggml/src/ggml-vulkan/ggml-vulkan.cpp index f615ae11a..120f484e5 100644 --- a/external/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/external/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -811,7 +811,8 @@ struct vk_device_struct { vk_pipeline pipeline_softplus[2]; vk_pipeline pipeline_step[2]; vk_pipeline pipeline_round[2]; - vk_pipeline pipeline_round_bf16[3]; + vk_pipeline pipeline_round_bf16[3]; + vk_pipeline pipeline_round_bf16_strided[3]; vk_pipeline pipeline_ceil[2]; vk_pipeline pipeline_floor[2]; vk_pipeline pipeline_trunc[2]; @@ -4754,7 +4755,11 @@ static void ggml_vk_load_shaders(vk_device& device) { // round-to-bf16: f32/f16/bf16 in, always f32 out (index by src type). ggml_vk_create_pipeline(device, device->pipeline_round_bf16[0], "round_bf16_f32", round_bf16_f32_len, round_bf16_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_round_bf16[1], "round_bf16_f16", round_bf16_f16_len, round_bf16_f16_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_round_bf16[2], "round_bf16_bf16", round_bf16_bf16_len, round_bf16_bf16_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_round_bf16[2], "round_bf16_bf16", round_bf16_bf16_len, round_bf16_bf16_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1); + // strided variant for non-contiguous (e.g. row-strided view) inputs. + ggml_vk_create_pipeline(device, device->pipeline_round_bf16_strided[0], "round_bf16_strided_f32", round_bf16_strided_f32_len, round_bf16_strided_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_round_bf16_strided[1], "round_bf16_strided_f16", round_bf16_strided_f16_len, round_bf16_strided_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_round_bf16_strided[2], "round_bf16_strided_bf16", round_bf16_strided_bf16_len, round_bf16_strided_bf16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f16, "add1_f16_f16", add1_f16_f16_len, add1_f16_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f32, "add1_f16_f32", add1_f16_f32_len, add1_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1); @@ -9751,10 +9756,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const if (dst->type != GGML_TYPE_F32) { return nullptr; } + const bool strided = !ggml_is_contiguous(src0) || !ggml_is_contiguous(dst); switch (src0->type) { - case GGML_TYPE_F32: return ctx->device->pipeline_round_bf16[0]; - case GGML_TYPE_F16: return ctx->device->pipeline_round_bf16[1]; - case GGML_TYPE_BF16: return ctx->device->pipeline_round_bf16[2]; + case GGML_TYPE_F32: return strided ? ctx->device->pipeline_round_bf16_strided[0] : ctx->device->pipeline_round_bf16[0]; + case GGML_TYPE_F16: return strided ? ctx->device->pipeline_round_bf16_strided[1] : ctx->device->pipeline_round_bf16[1]; + case GGML_TYPE_BF16: return strided ? ctx->device->pipeline_round_bf16_strided[2] : ctx->device->pipeline_round_bf16[2]; default: return nullptr; } } @@ -11499,6 +11505,11 @@ static void ggml_vk_sigmoid_strided(ggml_backend_vk_context * ctx, vk_context& s ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, std::move(p)); } +static void ggml_vk_round_bf16_strided(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, std::move(p)); +} + static void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, @@ -13534,13 +13545,19 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr case GGML_UNARY_OP_SOFTPLUS: case GGML_UNARY_OP_STEP: case GGML_UNARY_OP_ROUND: - case GGML_UNARY_OP_ROUND_BF16: case GGML_UNARY_OP_CEIL: case GGML_UNARY_OP_FLOOR: case GGML_UNARY_OP_TRUNC: case GGML_UNARY_OP_SGN: ggml_vk_unary(ctx, compute_ctx, src0, node); break; + case GGML_UNARY_OP_ROUND_BF16: + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(node)) { + ggml_vk_round_bf16_strided(ctx, compute_ctx, src0, node); + break; + } + ggml_vk_unary(ctx, compute_ctx, src0, node); + break; case GGML_UNARY_OP_SIGMOID: if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(node)) { ggml_vk_sigmoid_strided(ctx, compute_ctx, src0, node); @@ -15792,8 +15809,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && (op->src[0]->type == op->type); case GGML_UNARY_OP_ROUND_BF16: - return ggml_is_contiguous(op->src[0]) && - (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) && + return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) && (op->type == GGML_TYPE_F32); case GGML_UNARY_OP_SIGMOID: return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && diff --git a/external/ggml/src/ggml-vulkan/vulkan-shaders/round_bf16_strided.comp b/external/ggml/src/ggml-vulkan/vulkan-shaders/round_bf16_strided.comp new file mode 100644 index 000000000..d5eaa5087 --- /dev/null +++ b/external/ggml/src/ggml-vulkan/vulkan-shaders/round_bf16_strided.comp @@ -0,0 +1,23 @@ +#version 450 + +#include "types.glsl" +#include "generic_unary_head.glsl" + +layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in; + +void main() { + const uint idx = get_idx(); + + if (idx >= p.ne) { + return; + } + +#if defined(DATA_A_BF16) + const float x = bf16_to_fp32(uint32_t(data_a[get_aoffset() + src0_idx(idx)])); +#else + const float x = float(data_a[get_aoffset() + src0_idx(idx)]); +#endif + // Round to bf16 precision and widen back to f32, matching the + // f32 -> bf16 -> f32 cast round trip. + data_d[get_doffset() + dst_idx(idx)] = D_TYPE(bf16_to_fp32(fp32_to_bf16(x))); +} diff --git a/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 92fbe3e50..40c223470 100644 --- a/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -882,7 +882,10 @@ void process_shaders() { string_to_spv("round_f32", "round.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("round_bf16_f32", "round_bf16.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("round_bf16_f16", "round_bf16.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float"}}); - string_to_spv("round_bf16_bf16", "round_bf16.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "float"}, {"DATA_A_BF16", "1"}}); + string_to_spv("round_bf16_bf16", "round_bf16.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "float"}, {"DATA_A_BF16", "1"}}); + string_to_spv("round_bf16_strided_f32", "round_bf16_strided.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); + string_to_spv("round_bf16_strided_f16", "round_bf16_strided.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float"}}); + string_to_spv("round_bf16_strided_bf16", "round_bf16_strided.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "float"}, {"DATA_A_BF16", "1"}}); string_to_spv("ceil_f16", "ceil.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); string_to_spv("ceil_f32", "ceil.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}); string_to_spv("floor_f16", "floor.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});