Breeze encoder chunked vram

external/ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -811,6 +811,8 @@ struct vk_device_struct {
     vk_pipeline pipeline_softplus[2];
     vk_pipeline pipeline_step[2];
     vk_pipeline pipeline_round[2];
+    vk_pipeline pipeline_round_bf16[3];
+    vk_pipeline pipeline_round_bf16_strided[3];
     vk_pipeline pipeline_ceil[2];
     vk_pipeline pipeline_floor[2];
     vk_pipeline pipeline_trunc[2];
@@ -4750,6 +4752,15 @@ static void ggml_vk_load_shaders(vk_device& device) {
     CREATE_UNARY(exp)
 #undef CREATE_UNARY
 
+    // round-to-bf16: f32/f16/bf16 in, always f32 out (index by src type).
+    ggml_vk_create_pipeline(device, device->pipeline_round_bf16[0], "round_bf16_f32", round_bf16_f32_len, round_bf16_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
+    ggml_vk_create_pipeline(device, device->pipeline_round_bf16[1], "round_bf16_f16", round_bf16_f16_len, round_bf16_f16_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
+    ggml_vk_create_pipeline(device, device->pipeline_round_bf16[2], "round_bf16_bf16", round_bf16_bf16_len, round_bf16_bf16_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
+    // strided variant for non-contiguous (e.g. row-strided view) inputs.
+    ggml_vk_create_pipeline(device, device->pipeline_round_bf16_strided[0], "round_bf16_strided_f32", round_bf16_strided_f32_len, round_bf16_strided_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
+    ggml_vk_create_pipeline(device, device->pipeline_round_bf16_strided[1], "round_bf16_strided_f16", round_bf16_strided_f16_len, round_bf16_strided_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
+    ggml_vk_create_pipeline(device, device->pipeline_round_bf16_strided[2], "round_bf16_strided_bf16", round_bf16_strided_bf16_len, round_bf16_strided_bf16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
+
     ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f16, "add1_f16_f16", add1_f16_f16_len, add1_f16_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);
     ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f32, "add1_f16_f32", add1_f16_f32_len, add1_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);
     ggml_vk_create_pipeline(device, device->pipeline_add1_f32_f32, "add1_f32_f32", add1_f32_f32_len, add1_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);
@@ -9740,6 +9751,19 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
         }
         return nullptr;
     case GGML_OP_UNARY:
+        // ROUND_BF16 widens to f32: src may be f32/f16/bf16 while dst is f32.
+        if (ggml_get_unary_op(dst) == GGML_UNARY_OP_ROUND_BF16) {
+            if (dst->type != GGML_TYPE_F32) {
+                return nullptr;
+            }
+            const bool strided = !ggml_is_contiguous(src0) || !ggml_is_contiguous(dst);
+            switch (src0->type) {
+                case GGML_TYPE_F32:  return strided ? ctx->device->pipeline_round_bf16_strided[0] : ctx->device->pipeline_round_bf16[0];
+                case GGML_TYPE_F16:  return strided ? ctx->device->pipeline_round_bf16_strided[1] : ctx->device->pipeline_round_bf16[1];
+                case GGML_TYPE_BF16: return strided ? ctx->device->pipeline_round_bf16_strided[2] : ctx->device->pipeline_round_bf16[2];
+                default:             return nullptr;
+            }
+        }
         if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) ||
             (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) ||
             (src0->type != dst->type)) {
@@ -11481,6 +11505,11 @@ static void ggml_vk_sigmoid_strided(ggml_backend_vk_context * ctx, vk_context& s
     ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, std::move(p));
 }
 
+static void ggml_vk_round_bf16_strided(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
+    vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
+    ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY, std::move(p));
+}
+
 static void ggml_vk_xielu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
     float * op_params = (float *)dst->op_params;
     ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_UNARY,
@@ -13522,6 +13551,13 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr
         case GGML_UNARY_OP_SGN:
             ggml_vk_unary(ctx, compute_ctx, src0, node);
             break;
+        case GGML_UNARY_OP_ROUND_BF16:
+            if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(node)) {
+                ggml_vk_round_bf16_strided(ctx, compute_ctx, src0, node);
+                break;
+            }
+            ggml_vk_unary(ctx, compute_ctx, src0, node);
+            break;
         case GGML_UNARY_OP_SIGMOID:
             if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(node)) {
                 ggml_vk_sigmoid_strided(ctx, compute_ctx, src0, node);
@@ -15772,6 +15808,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
                            (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
                            (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) &&
                            (op->src[0]->type == op->type);
+                case GGML_UNARY_OP_ROUND_BF16:
+                    return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) &&
+                           (op->type == GGML_TYPE_F32);
                 case GGML_UNARY_OP_SIGMOID:
                     return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
                            (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) &&

external/ggml/src/ggml-vulkan/vulkan-shaders/round_bf16.comp

@@ -0,0 +1,26 @@
+#version 450
+
+#include "generic_head.glsl"
+#include "types.glsl"
+
+layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
+
+layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
+layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
+
+void main() {
+    const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
+
+    if (i >= p.KX) {
+        return;
+    }
+
+#if defined(DATA_A_BF16)
+    const float x = bf16_to_fp32(uint32_t(data_a[i]));
+#else
+    const float x = float(data_a[i]);
+#endif
+    // Round to bf16 precision and widen back to f32, matching the
+    // f32 -> bf16 -> f32 cast round trip.
+    data_d[i] = D_TYPE(bf16_to_fp32(fp32_to_bf16(x)));
+}

external/ggml/src/ggml-vulkan/vulkan-shaders/round_bf16_strided.comp

@@ -0,0 +1,23 @@
+#version 450
+
+#include "types.glsl"
+#include "generic_unary_head.glsl"
+
+layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
+
+void main() {
+    const uint idx = get_idx();
+
+    if (idx >= p.ne) {
+        return;
+    }
+
+#if defined(DATA_A_BF16)
+    const float x = bf16_to_fp32(uint32_t(data_a[get_aoffset() + src0_idx(idx)]));
+#else
+    const float x = float(data_a[get_aoffset() + src0_idx(idx)]);
+#endif
+    // Round to bf16 precision and widen back to f32, matching the
+    // f32 -> bf16 -> f32 cast round trip.
+    data_d[get_doffset() + dst_idx(idx)] = D_TYPE(bf16_to_fp32(fp32_to_bf16(x)));
+}

external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp

@@ -880,6 +880,12 @@ void process_shaders() {
     string_to_spv("step_f32",       "step.comp",        {{"A_TYPE", "float"},       {"D_TYPE", "float"}});
     string_to_spv("round_f16",      "round.comp",       {{"A_TYPE", "float16_t"},   {"D_TYPE", "float16_t"}});
     string_to_spv("round_f32",      "round.comp",       {{"A_TYPE", "float"},       {"D_TYPE", "float"}});
+    string_to_spv("round_bf16_f32",  "round_bf16.comp", {{"A_TYPE", "float"},       {"D_TYPE", "float"}});
+    string_to_spv("round_bf16_f16",  "round_bf16.comp", {{"A_TYPE", "float16_t"},   {"D_TYPE", "float"}});
+    string_to_spv("round_bf16_bf16", "round_bf16.comp", {{"A_TYPE", "uint16_t"},    {"D_TYPE", "float"}, {"DATA_A_BF16", "1"}});
+    string_to_spv("round_bf16_strided_f32",  "round_bf16_strided.comp", {{"A_TYPE", "float"},     {"D_TYPE", "float"}});
+    string_to_spv("round_bf16_strided_f16",  "round_bf16_strided.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float"}});
+    string_to_spv("round_bf16_strided_bf16", "round_bf16_strided.comp", {{"A_TYPE", "uint16_t"},  {"D_TYPE", "float"}, {"DATA_A_BF16", "1"}});
     string_to_spv("ceil_f16",       "ceil.comp",        {{"A_TYPE", "float16_t"},   {"D_TYPE", "float16_t"}});
     string_to_spv("ceil_f32",       "ceil.comp",        {{"A_TYPE", "float"},       {"D_TYPE", "float"}});
     string_to_spv("floor_f16",      "floor.comp",       {{"A_TYPE", "float16_t"},   {"D_TYPE", "float16_t"}});

include/engine/models/breeze_tts/speech_encoder.h

@@ -19,7 +19,8 @@ namespace engine::models {
 namespace breeze_tts {
 
 struct BreezeSpeechEncoderWeights;
-class BreezeSpeechEncoderGraph;
+class BreezeSpeechEncoderConvGraph;
+class BreezeSpeechEncoderTransformerGraph;
 
 struct BreezeSpeechEncoderOutput {
     BreezeSpeechCodes codes;
@@ -46,7 +47,8 @@ class BreezeSpeechEncoderRuntime {
     core::ExecutionContext * execution_context_ = nullptr;
     size_t graph_arena_bytes_ = 0;
     std::unique_ptr<core::ConstantTensorCache> constants_;
-    mutable std::unique_ptr<BreezeSpeechEncoderGraph> graph_;
+    mutable std::unique_ptr<BreezeSpeechEncoderConvGraph> conv_graph_;
+    mutable std::unique_ptr<BreezeSpeechEncoderTransformerGraph> transformer_graph_;
 };
 
 }  // namespace breeze_tts

src/models/breeze_tts/generator.cpp

@@ -60,9 +60,9 @@ modules::QwenDecoderActivationCastPolicy breeze_bf16_activation_policy(core::Bac
     }
     policy.enabled = true;
     policy.type = GGML_TYPE_BF16;
-    // CUDA/HIP implement the fused round-to-bf16 unary op; Vulkan does not and
-    // keeps the cast round trip.
-    policy.fused_round = backend_type == core::BackendType::Cuda || backend_type == core::BackendType::Hip;
+    // CUDA/HIP/Vulkan implement the fused round-to-bf16 unary op.
+    policy.fused_round = backend_type == core::BackendType::Cuda || backend_type == core::BackendType::Hip ||
+        backend_type == core::BackendType::Vulkan;
     policy.after_input_norm = true;
     policy.after_qkv_projection = true;
     policy.after_qk_norm = true;