fix(attention): fallback for GPUs without flash MMA kernels (sm70)

CMakeLists.txt

@@ -353,6 +353,7 @@ add_library(engine_core OBJECT
     src/framework/assets/torch_bin.cpp
     src/framework/core/module.cpp
     src/framework/core/backend.cpp
+    src/framework/core/attention_fallback.cpp
     src/framework/core/deferred_tensor_writer.cpp
     src/framework/core/execution_context.cpp
     src/framework/core/host_memory.cpp
@@ -2213,6 +2214,14 @@ if (ENGINE_BUILD_TESTS)
         COMMAND backend_device_resolution_test
     )
 
+    add_engine_unittest(attention_fallback_test tests/unittests/test_attention_fallback.cpp)
+    target_include_directories(attention_fallback_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests)
+
+    add_test(
+        NAME attention_fallback_test
+        COMMAND attention_fallback_test
+    )
+
     add_executable(streaming_audio_input_test
         tests/unittests/test_streaming_audio_input.cpp
         app/streaming/pcm_source.cpp

docs/models/breeze_tts.md

@@ -67,6 +67,7 @@ audiocpp_cli \
 | `--request-option top_p=<f>` | `0..1` | `1.0` | Top-p sampling limit. |
 | `--request-option seed=<n>` | integer >= 0 | `0` | Generation seed. |
 | `--session-option breeze_tts.reference_cache_slots=<n>` | integer >= 0 | `1` | Prepared reference-audio cache slots. |
+| `--session-option breeze_tts.attention=<mode>` | `auto`, `flash`, `eager` | `auto` | Attention kernel. `auto` uses flash except on Volta/Turing GPUs (e.g. V100), where it falls back to eager to avoid missing MMA kernels. |
 | `--session-option weight_type=<type>` | `native`, `f32`, `f16`, `bf16`, `q8_0`, `q4_0`, `q4_k` | `native` | Weight storage type; quantized types convert at load time from the BF16 package. |
 
 Quantized weight storage is the largest measured speedup and applies to CUDA
@@ -76,4 +77,3 @@ and HIP alike: `q8_0` cut the fixed 100-token regression case from RTF ~1.5 to
 voice-design regression cases. Counter to intuition, fp32 is the one
 configuration known to be *worse* for this model (mispronunciations and
 runaway repetition), because the model is trained and tuned in bf16.
-

docs/tts.md

@@ -559,6 +559,7 @@ python3 tools/model_manager_v2.py install --models-root models higgs_audio_tts_4
 | `--top-k` | integer | `30` | AR top-k sampling limit. The narrower default is less prone to premature EOC than the Python client's `50`. |
 | `--top-p` | float | `0.8` | AR nucleus sampling limit. The Python client's unfiltered equivalent is `1.0`. |
 | `--repetition-penalty` | float | `1.1` | Accepted for Python API compatibility; Higgs audio-code sampling does not consume it. |
+| `--session-option higgs_audio_tts.attention=<mode>` | `auto`, `flash`, `eager` | `auto` | Attention kernel. `auto` uses flash except on Volta/Turing GPUs (e.g. V100), where it falls back to eager to avoid missing MMA kernels. |
 
 ## Fish Audio S2 Pro
 

include/engine/framework/core/attention_fallback.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#include <cstdint>
+#include <string>
+
+namespace engine::core {
+
+// Session-option vocabulary shared by families that lower attention with
+// ggml_flash_attn_ext: "auto" (default), "flash", or "eager".
+//
+// Background: ggml-cuda only instantiates the MMA/wmma flash-attention kernels
+// for compute capability >= 8.0 (Ampere). On older GPUs such as Volta/sm70
+// (e.g. Tesla V100) a graph containing GGML_OP_FLASH_ATTN_EXT fails at compute
+// time with "no device code compatible with CUDA arch 700", even when ggml was
+// built with that arch enabled. The eager (explicit matmul + softmax) lowering
+// computes the same operation with generic ops and runs everywhere (output
+// logits may differ at ulp level, as with any kernel change).
+enum class AttentionPreference {
+    Auto,
+    Flash,
+    Eager,
+};
+
+// Parses a "<family>.attention" session-option value. Throws std::runtime_error
+// naming option_name on invalid input.
+AttentionPreference parse_attention_preference(const std::string & value, const char * option_name);
+
+// Resolves whether flash attention may be used for the given backend and head
+// dimension. Flash forces true, Eager forces false, Auto gates on the CUDA
+// device compute capability (Volta/Turing resolve to eager; see the .cpp for
+// why supports_op cannot be used). A null or non-CUDA backend, or a device
+// query failure, preserves historical behavior (true).
+//
+// Adopting in other families (currently wired for higgs_audio_tts and
+// breeze_tts only): resolve once per runtime with the model's head_dim and
+// the family's "<family>.attention" session option, then switch the
+// QwenDecoder prefill/static modes (or SDPA/GQA lowerings) between flash and
+// their ManualRepeat/Explicit equivalents based on the result.
+bool resolve_flash_attention(ggml_backend_t backend, int64_t head_dim, AttentionPreference preference);
+
+}  // namespace engine::core

include/engine/framework/modules/transformers/qwen_decoder.h

@@ -77,6 +77,9 @@ struct QwenDecoderAttentionPolicy {
     QwenDecoderAttentionMode static_mode = QwenDecoderAttentionMode::FlashGrouped;
     QwenDecoderPrefixAttentionMode prefix_mode = QwenDecoderPrefixAttentionMode::Exact;
     int64_t grouped_query_min_steps = 0;
+    // False routes flash branches through repeat-KV + matmul/softmax for GPUs
+    // without a flash kernel (e.g. CUDA sm70). True preserves historical behavior.
+    bool allow_flash_attention = true;
 };
 
 struct QwenDecoderStaticCachePolicy {

include/engine/models/breeze_tts/generator.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "engine/framework/core/attention_fallback.h"
 #include "engine/framework/runtime/session.h"
 #include "engine/models/breeze_tts/assets.h"
 #include "engine/models/breeze_tts/speech_decoder.h"
@@ -35,7 +36,8 @@ class BreezeGeneratorRuntime {
         engine::core::ExecutionContext & execution,
         size_t graph_arena_bytes,
         size_t weight_context_bytes,
-        engine::assets::TensorStorageType storage_type);
+        engine::assets::TensorStorageType storage_type,
+        engine::core::AttentionPreference attention_preference = engine::core::AttentionPreference::Auto);
     ~BreezeGeneratorRuntime();
 
     engine::runtime::AudioBuffer generate(const BreezeGenerationRequest & request);

include/engine/models/breeze_tts/speech_decoder.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "engine/framework/core/attention_fallback.h"
 #include "engine/framework/core/execution_context.h"
 #include "engine/framework/assets/tensor_source.h"
 #include "engine/framework/runtime/session.h"
@@ -36,7 +37,8 @@ class BreezeSpeechDecoderRuntime {
         size_t graph_arena_bytes,
         size_t constant_context_bytes,
         engine::assets::TensorStorageType linear_weight_storage_type,
-        engine::assets::TensorStorageType conv_weight_storage_type);
+        engine::assets::TensorStorageType conv_weight_storage_type,
+        core::AttentionPreference attention_preference = core::AttentionPreference::Auto);
     ~BreezeSpeechDecoderRuntime();
 
     runtime::AudioBuffer decode(const BreezeSpeechCodes & codec_codes) const;
@@ -50,6 +52,7 @@ class BreezeSpeechDecoderRuntime {
     core::ExecutionContext * execution_context_ = nullptr;
     std::shared_ptr<const BreezeSpeechDecoderWeights> weights_;
     size_t graph_arena_bytes_ = 0;
+    bool allow_flash_attention_ = true;
     std::unique_ptr<core::ConstantTensorCache> constants_;
     mutable std::unique_ptr<BreezeSpeechDecoderGraph> graph_;
     // Always present to keep this public class layout identical when the private

include/engine/models/breeze_tts/speech_encoder.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "engine/framework/core/attention_fallback.h"
 #include "engine/framework/core/execution_context.h"
 #include "engine/framework/assets/tensor_source.h"
 #include "engine/framework/runtime/session.h"
@@ -35,7 +36,8 @@ class BreezeSpeechEncoderRuntime {
         core::ExecutionContext & execution_context,
         size_t graph_arena_bytes,
         engine::assets::TensorStorageType linear_weight_storage_type,
-        engine::assets::TensorStorageType conv_weight_storage_type);
+        engine::assets::TensorStorageType conv_weight_storage_type,
+        core::AttentionPreference attention_preference = core::AttentionPreference::Auto);
     ~BreezeSpeechEncoderRuntime();
 
     BreezeSpeechCodes encode(const runtime::AudioBuffer & audio) const;
@@ -46,6 +48,7 @@ class BreezeSpeechEncoderRuntime {
     std::shared_ptr<const BreezeSpeechEncoderWeights> weights_;
     core::ExecutionContext * execution_context_ = nullptr;
     size_t graph_arena_bytes_ = 0;
+    bool allow_flash_attention_ = true;
     std::unique_ptr<core::ConstantTensorCache> constants_;
     mutable std::unique_ptr<BreezeSpeechEncoderConvGraph> conv_graph_;
     mutable std::unique_ptr<BreezeSpeechEncoderTransformerGraph> transformer_graph_;

include/engine/models/higgs_audio_tts/ar.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "engine/framework/assets/tensor_source.h"
+#include "engine/framework/core/attention_fallback.h"
 #include "engine/framework/core/execution_context.h"
 #include "engine/framework/core/module.h"
 #include "engine/framework/modules/transformers/qwen_decoder.h"
@@ -44,21 +45,24 @@ class HiggsARRuntime {
         std::shared_ptr<const HiggsAssets> assets,
         core::ExecutionContext & execution,
         size_t weight_context_bytes,
-        assets::TensorStorageType weight_storage_type);
+        assets::TensorStorageType weight_storage_type,
+        core::AttentionPreference attention_preference = core::AttentionPreference::Auto);
 
     const HiggsAssets & assets() const noexcept;
     const HiggsARWeights & weights() const noexcept;
     ggml_backend_t backend() const noexcept;
     core::BackendType backend_type() const noexcept;
     int device() const noexcept;
     int threads() const noexcept;
+    bool allow_flash_attention() const noexcept;
 
 private:
     std::shared_ptr<const HiggsAssets> assets_;
     ggml_backend_t backend_ = nullptr;
     core::BackendType backend_type_ = core::BackendType::Cpu;
     int device_ = 0;
     int threads_ = 1;
+    bool allow_flash_attention_ = true;
     std::shared_ptr<const HiggsARWeights> weights_;
 };
 

model_specs/breeze_tts.json

@@ -161,6 +161,14 @@
         "required": false,
         "min": 0,
         "default": 1
+      },
+      {
+        "name": "attention",
+        "type": "enum",
+        "description": "Attention lowering; auto probes the backend and falls back to eager on GPUs without a flash kernel (e.g. sm70); default auto.",
+        "required": false,
+        "values": ["auto", "flash", "eager"],
+        "default": "auto"
       }
     ],
     "load": []

src/framework/core/attention_fallback.cpp

@@ -0,0 +1,121 @@
+#include "engine/framework/core/attention_fallback.h"
+
+#include <algorithm>
+#include <cctype>
+#include <cstdlib>
+#include <cstring>
+#include <stdexcept>
+#include <string>
+
+#include "ggml.h"
+#include "ggml-backend.h"
+
+#ifdef GGML_USE_CUDA
+// CUDA driver API, declared manually so this translation unit needs neither
+// the CUDA headers on its include path nor any CMake changes. The driver
+// library is already linked transitively through ggml-cuda. Attribute ids
+// CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR=75 / MINOR=76 are stable ABI.
+extern "C" {
+typedef int kCcProbeCuDevice;
+typedef int kCcProbeCuResult;
+kCcProbeCuResult cuDeviceGet(kCcProbeCuDevice * device, int ordinal);
+kCcProbeCuResult cuDeviceGetAttribute(int * value, int attrib, kCcProbeCuDevice device);
+}
+#endif  // GGML_USE_CUDA
+
+namespace engine::core {
+namespace {
+
+std::string to_lower(std::string value) {
+    std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) {
+        return static_cast<char>(std::tolower(c));
+    });
+    return value;
+}
+
+AttentionPreference parse_preference_value(const std::string & value, const char * option_name) {
+    const std::string lowered = to_lower(value);
+    if (lowered == "auto") {
+        return AttentionPreference::Auto;
+    }
+    if (lowered == "flash" || lowered == "on" || lowered == "1") {
+        return AttentionPreference::Flash;
+    }
+    if (lowered == "eager" || lowered == "off" || lowered == "0") {
+        return AttentionPreference::Eager;
+    }
+    throw std::runtime_error(
+        std::string(option_name) + " must be 'auto', 'flash', or 'eager' (got '" + value + "')");
+}
+
+// Auto-resolution for the CUDA flash-attention path.
+//
+// ggml_backend_supports_op() cannot be used here: on Volta it returns true
+// (the MMA kernel is "selected") yet large prefill shapes die at launch with
+// "flash_attn_ext_f16 has no device code compatible with CUDA arch 700".
+// Instead, gate on compute capability, mirroring the kernel guards in
+// ggml-cuda: flash below 700 (only generic TILE/VEC kernels exist) and at or
+// above 800 (MMA fully instantiated); eager on 700-800, where large shapes
+// select the MMA kernel with no usable device code. Unknown backends and
+// query failures fail OPEN to preserve current behavior.
+bool cuda_device_wants_eager(ggml_backend_t backend) {
+#ifdef GGML_USE_CUDA
+    if (backend == nullptr) {
+        return false;
+    }
+    ggml_backend_dev_t device = ggml_backend_get_device(backend);
+    if (device == nullptr) {
+        return false;
+    }
+    if (ggml_backend_dev_type(device) != GGML_BACKEND_DEVICE_TYPE_GPU) {
+        return false;
+    }
+    const char * name = ggml_backend_dev_name(device);
+    if (name == nullptr || std::strncmp(name, "CUDA", 4) != 0) {
+        return false;  // HIP / Vulkan / Metal / CPU: unchanged behavior.
+    }
+    char * end = nullptr;
+    const long ordinal = std::strtol(name + 4, &end, 10);
+    if (end == name + 4 || ordinal < 0) {
+        return false;
+    }
+    kCcProbeCuDevice cu_device = -1;
+    if (cuDeviceGet(&cu_device, static_cast<int>(ordinal)) != 0) {
+        return false;
+    }
+    int major = 0;
+    int minor = 0;
+    if (cuDeviceGetAttribute(&major, 75 /* COMPUTE_CAPABILITY_MAJOR */, cu_device) != 0) {
+        return false;
+    }
+    if (cuDeviceGetAttribute(&minor, 76 /* COMPUTE_CAPABILITY_MINOR */, cu_device) != 0) {
+        return false;
+    }
+    const int cc = major * 100 + minor * 10;
+    return cc >= 700 && cc < 800;
+#else
+    (void) backend;
+    return false;
+#endif  // GGML_USE_CUDA
+}
+
+}  // namespace
+
+AttentionPreference parse_attention_preference(const std::string & value, const char * option_name) {
+    return parse_preference_value(value, option_name != nullptr ? option_name : "attention");
+}
+
+bool resolve_flash_attention(ggml_backend_t backend, int64_t head_dim, AttentionPreference preference) {
+    (void) head_dim;
+    switch (preference) {
+        case AttentionPreference::Flash:
+            return true;
+        case AttentionPreference::Eager:
+            return false;
+        case AttentionPreference::Auto:
+            break;
+    }
+    return !cuda_device_wants_eager(backend);
+}
+
+}  // namespace engine::core