Add livekit-plugins-funasr (FunASR/SenseVoice local STT)

livekit-agents/livekit/agents/stt/multi_speaker_adapter.py

@@ -18,6 +18,7 @@
 from .stt import STT, RecognizeStream, SpeechData, SpeechEvent, SpeechEventType
 
 if TYPE_CHECKING:
+    from ..llm.chat_context import MetricsMetadata
     from ..voice.events import ConversationItemAddedEvent
 
 
@@ -79,6 +80,10 @@ def model(self) -> str:
     def provider(self) -> str:
         return self._stt.provider
 
+    @property
+    def metrics_metadata(self) -> MetricsMetadata:
+        return self._stt.metrics_metadata
+
     def _update_session_keyterms(self, keyterms: list[str]) -> None:
         self._stt._update_session_keyterms(keyterms)
 

livekit-agents/livekit/agents/stt/stream_adapter.py

@@ -11,6 +11,7 @@
 from .stt import STT, RecognizeStream, SpeechEvent, SpeechEventType, STTCapabilities
 
 if TYPE_CHECKING:
+    from ..llm.chat_context import MetricsMetadata
     from ..voice.events import ConversationItemAddedEvent
 
 # already a retry mechanism in STT.recognize, don't retry in stream adapter
@@ -48,6 +49,10 @@ def model(self) -> str:
     def provider(self) -> str:
         return self._stt.provider
 
+    @property
+    def metrics_metadata(self) -> MetricsMetadata:
+        return self._stt.metrics_metadata
+
     def _update_session_keyterms(self, keyterms: list[str]) -> None:
         self._stt._update_session_keyterms(keyterms)
 

livekit-agents/livekit/agents/stt/stt.py

@@ -197,7 +197,7 @@ def provider(self) -> str:
 
     @property
     def metrics_metadata(self) -> MetricsMetadata:
-        """Metadata used to label turn metrics emitted for this STT instance."""
+        """Metadata used to label metrics, traces, and error events for this STT instance."""
         return {"model_name": self.model, "model_provider": self.provider}
 
     @property
@@ -235,10 +235,7 @@ async def recognize(
                         label=self._label,
                         audio_duration=calculate_audio_duration(buffer),
                         streamed=False,
-                        metadata=Metadata(
-                            model_name=self.model,
-                            model_provider=self.provider,
-                        ),
+                        metadata=Metadata(**self.metrics_metadata),
                     )
                     self.emit("metrics_collected", stt_metrics)
                 return event
@@ -455,7 +452,7 @@ def _report_connection_acquired(self, acquire_time: float, connection_reused: bo
                 streamed=True,
                 acquire_time=acquire_time,
                 connection_reused=connection_reused,
-                metadata=Metadata(model_name=self._stt.model, model_provider=self._stt.provider),
+                metadata=Metadata(**self._stt.metrics_metadata),
             ),
         )
 
@@ -532,9 +529,7 @@ async def _metrics_monitor_task(self, event_aiter: AsyncIterable[SpeechEvent]) -
                     input_tokens=ev.recognition_usage.input_tokens,
                     output_tokens=ev.recognition_usage.output_tokens,
                     streamed=True,
-                    metadata=Metadata(
-                        model_name=self._stt.model, model_provider=self._stt.provider
-                    ),
+                    metadata=Metadata(**self._stt.metrics_metadata),
                 )
 
                 self._stt.emit("metrics_collected", stt_metrics)

livekit-agents/livekit/agents/voice/agent_activity.py

@@ -748,10 +748,15 @@ def _update_models(
             self._agent._stt = new_stt
             resolved_stt = self.stt
             if self._audio_recognition is not None:
+                stt_metadata = (
+                    Metadata(**resolved_stt.metrics_metadata)
+                    if isinstance(resolved_stt, stt.STT)
+                    else Metadata()
+                )
                 self._audio_recognition._update_stt(
                     self._agent.stt_node if resolved_stt else None,
-                    model=resolved_stt.model if isinstance(resolved_stt, stt.STT) else None,
-                    provider=resolved_stt.provider if isinstance(resolved_stt, stt.STT) else None,
+                    model=stt_metadata.model_name,
+                    provider=stt_metadata.model_provider,
                     aligned_transcript=bool(resolved_stt.capabilities.aligned_transcript)
                     if isinstance(resolved_stt, stt.STT)
                     else False,
@@ -1160,6 +1165,7 @@ async def _start_session(self, *, reuse_resources: _ReusableResources | None = N
         wired_vad = self.vad
         if wired_vad is not None and self.using_default_vad and self._rt_turn_detection_enabled:
             wired_vad = None
+        stt_metadata = Metadata(**self.stt.metrics_metadata) if self.stt else Metadata()
         self._audio_recognition = AudioRecognition(
             self._session,
             hooks=self,
@@ -1168,8 +1174,13 @@ async def _start_session(self, *, reuse_resources: _ReusableResources | None = N
             interruption_detection=self._interruption_detector,
             endpointing=create_endpointing(self.endpointing_opts),
             turn_detection=self._turn_detection,
-            stt_model=self.stt.model if self.stt else None,
-            stt_provider=self.stt.provider if self.stt else None,
+            stt_model=stt_metadata.model_name,
+            stt_provider=stt_metadata.model_provider,
+            stt_metadata=lambda: (
+                Metadata(**current_stt.metrics_metadata)
+                if (current_stt := self.stt) is not None
+                else Metadata()
+            ),
             stt_aligned_transcript=bool(self.stt.capabilities.aligned_transcript)
             if self.stt
             else False,

livekit-agents/livekit/agents/voice/amd/detector.py

@@ -222,6 +222,7 @@ def __init__(
                     self._stt.model,
                     EVALUATED_STT_MODELS,
                     model_kind="stt",
+                    display_model=self._stt.metrics_metadata.get("model_name"),
                 )
 
         self._setup_task: asyncio.Task[None] | None = None
@@ -630,6 +631,7 @@ def _warn_if_not_evaluated(
     evaluated_models: set[str],
     *,
     model_kind: str,
+    display_model: NotGivenOr[str | None] = NOT_GIVEN,
 ) -> None:
     if not model:
         return
@@ -643,5 +645,5 @@ def _warn_if_not_evaluated(
             "%s model %s hasn't been evaluated with our benchmark, it might not be compatible "
             "with amd. Set `suppress_compatibility_warning=True` to silence this warning.",
             model_kind,
-            model,
+            model if not is_given(display_model) else display_model or "unknown",
         )

livekit-agents/livekit/agents/voice/audio_recognition.py

@@ -26,6 +26,7 @@
 )
 from ..language import LanguageCode
 from ..log import logger
+from ..metrics.base import Metadata
 from ..stt import SpeechEvent
 from ..telemetry import trace_types, tracer
 from ..types import NOT_GIVEN, NotGivenOr
@@ -249,6 +250,7 @@ def __init__(
         turn_detection: TurnDetectionMode | None,
         stt_model: str | None = None,
         stt_provider: str | None = None,
+        stt_metadata: Callable[[], Metadata] | None = None,
         stt_aligned_transcript: bool = False,
     ) -> None:
         self._session = session
@@ -264,6 +266,7 @@ def __init__(
         self._vad = vad
         self._stt_model = stt_model
         self._stt_provider = stt_provider
+        self._stt_metadata = stt_metadata
         self._stt_aligned_transcript = stt_aligned_transcript
         self._turn_detection_mode = turn_detection if isinstance(turn_detection, str) else None
         self._vad_base_turn_detection = self._turn_detection_mode in ("vad", None)
@@ -1944,14 +1947,19 @@ def _ensure_user_turn_span(self, start_time: float | None = None) -> trace.Span:
         if (room_io := self._session._room_io) and room_io.linked_participant:
             _set_participant_attributes(self._user_turn_span, room_io.linked_participant)
 
-        # add STT model/provider attributes
-        if self._stt_model:
+        # Resolve at span creation: fallback adapters can switch without an STT hot swap.
+        metadata = (
+            self._stt_metadata()
+            if self._stt_metadata is not None
+            else Metadata(model_name=self._stt_model, model_provider=self._stt_provider)
+        )
+        if metadata.model_name:
             self._user_turn_span.set_attribute(
-                trace_types.ATTR_GEN_AI_REQUEST_MODEL, self._stt_model
+                trace_types.ATTR_GEN_AI_REQUEST_MODEL, metadata.model_name
             )
-        if self._stt_provider:
+        if metadata.model_provider:
             self._user_turn_span.set_attribute(
-                trace_types.ATTR_GEN_AI_PROVIDER_NAME, self._stt_provider
+                trace_types.ATTR_GEN_AI_PROVIDER_NAME, metadata.model_provider
             )
 
         return self._user_turn_span

livekit-agents/livekit/agents/voice/events.py

@@ -552,7 +552,10 @@ class ErrorEvent(BaseModel):
 
     @field_serializer("source")
     def _serialize_source(self, source: Any) -> Any:
-        if isinstance(source, LLM | STT | TTS | RealtimeModel | AdaptiveInterruptionDetector):
+        if isinstance(source, STT):
+            metadata = source.metrics_metadata
+            return {"model": metadata.get("model_name"), "provider": metadata.get("model_provider")}
+        if isinstance(source, LLM | TTS | RealtimeModel | AdaptiveInterruptionDetector):
             return {"model": source.model, "provider": source.provider}
         if isinstance(source, BaseModel):
             return source.model_dump()

livekit-agents/pyproject.toml

@@ -85,6 +85,7 @@ deepgram = ["livekit-plugins-deepgram>=1.8.0"]
 elevenlabs = ["livekit-plugins-elevenlabs>=1.8.0"]
 fal = ["livekit-plugins-fal>=1.8.0"]
 fishaudio = ["livekit-plugins-fishaudio>=1.8.0"]
+funasr = ["livekit-plugins-funasr>=1.8.0"]
 fireworksai = ["livekit-plugins-fireworksai>=1.8.0"]
 gladia = ["livekit-plugins-gladia>=1.8.0"]
 gnani = ["livekit-plugins-gnani>=1.8.0"]

livekit-plugins/livekit-plugins-funasr/README.md

@@ -0,0 +1,25 @@
+# LiveKit Plugins FunASR
+
+Agent Framework plugin for local speech-to-text with [FunASR](https://github.com/modelscope/FunASR) models such as [SenseVoice](https://github.com/FunAudioLLM/SenseVoice).
+
+SenseVoice is an open-source, fully-local, non-autoregressive multilingual ASR model (Chinese, Cantonese, English, Japanese, Korean and more) with leading Chinese accuracy and fast inference. The model runs locally, so no API key is required.
+
+## Installation
+
+```bash
+pip install livekit-plugins-funasr
+```
+
+## Usage
+
+```python
+from livekit.plugins import funasr
+
+stt = funasr.STT(model="iic/SenseVoiceSmall")
+```
+
+The first run downloads the model from ModelScope/Hugging Face. Use `language=None` (default) for automatic language detection, or set e.g. `language="zh"`.
+
+CPU inference works with the default installation. For GPU inference, install a
+PyTorch and torchaudio build that matches your CUDA runtime, then pass
+`device="cuda"`.

livekit-plugins/livekit-plugins-funasr/livekit/plugins/funasr/__init__.py

@@ -0,0 +1,44 @@
+# Copyright 2024 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""FunASR plugin for LiveKit Agents.
+
+Local, fully-offline multilingual speech-to-text using FunASR models such as
+SenseVoice (Chinese, Cantonese, English, Japanese, Korean and more).
+See https://github.com/modelscope/FunASR for more information.
+"""
+
+from .stt import _DEFAULT_MODEL, FunASRSTT, FunASRSTT as STT, _load_model
+from .version import __version__
+
+__all__ = ["FunASRSTT", "STT", "__version__"]
+
+from livekit.agents import Plugin
+
+from .log import logger
+
+
+class FunASRPlugin(Plugin):
+    """Register the FunASR integration and its model-download hook."""
+
+    def __init__(self) -> None:
+        """Create the LiveKit plugin registration."""
+        super().__init__(__name__, __version__, __package__, logger)
+
+    def download_files(self) -> None:
+        """Download the default SenseVoice model for offline agent startup."""
+        _load_model(_DEFAULT_MODEL, "cpu")
+
+
+Plugin.register_plugin(FunASRPlugin())

livekit-plugins/livekit-plugins-funasr/livekit/plugins/funasr/log.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("livekit.plugins.funasr")