diff --git a/CLAUDE.md b/CLAUDE.md index fbb095b..faa321e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,6 +6,16 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co AI Interview Platform (智能 AI 面试官平台) — a full-stack application using LLMs for resume analysis and simulated technical interviews. Python 3.11+, FastAPI backend + React 18 frontend. +## Git Workflow + +**新功能先切到 develop 分支，不要直接在 main 上改。** 详细规范见 [`git-workflow.md`](git-workflow.md)（分支策略、commit 格式、合并策略）。要点速记： +- 永久分支：`main`（生产）、`develop`（集成） +- 功能分支：`feat/*`、`fix/*`、`refactor/*`、`chore/*` — 都从 develop 切出 +- 热修例外：`hotfix/*` 从 main 切出，同时 merge 回 main + develop +- Commit 格式：`(): `，例如 `feat(interview): 新增动态面试复盘` +- 功能分支合入 develop 用 `--no-ff` merge；功能分支同步 develop 用 rebase +- 发版：`develop` → `release/` → `main`（`--no-ff` + tag）→ 同步回 develop + ## Commands ### Backend (local dev) @@ -106,6 +116,21 @@ Interview directions defined in `skills//` with: - Embedding: Zhipu Embedding-3 (2048-dim, truncated to 1536 for pgvector), DashScope fallback, hash-vector ultimate fallback - Prompt templates: markdown files in `app/prompts/` — paired `*-system.md` / `*-user.md` for each use case +## Voice / STT + +数字人面试场景的语音输入链路。两条模式： + +- **流式（主用）**：`POST /api/interview/voice/stream` 不存在；改用 `WS /api/interview/voice/stream?token=` + - 后端：[`app/modules/interview/ws_router.py`](app/modules/interview/ws_router.py) + [`voice_streaming_service.py`](app/modules/interview/voice_streaming_service.py)（FunASR SenseVoice-Small） + - 前端：浏览器 `AudioWorklet` (`public/audio-worklets/pcm-capture.js`) 抓 PCM → 200ms 切片 → `WebSocket.send` Int16 LE 16kHz + - hook：[`useVoiceInput.ts`](frontend/src/hooks/useVoiceInput.ts) 统一管理 AudioContext / WS / 状态机 + - 状态：`idle → streaming → idle`（或 `error`） +- **整段（fallback）**：`POST /api/interview/voice/transcribe` 仍保留，用 faster-whisper small + CPU + int8 整段转写 + +WS 鉴权：HTTP 的 `auth_middleware` 不覆盖 WebSocket，需在 handler 内手动 `decode_access_token(token)`，失败用 close code 1008。 + +详见 [docs/research/stt-selection-report.json](docs/research/stt-selection-report.json) 的选型分析。 + ## Environment Variables All config via `.env` file (Pydantic Settings). Key groups: diff --git a/app/common/error_code.py b/app/common/error_code.py index 84d1951..a334e8d 100644 --- a/app/common/error_code.py +++ b/app/common/error_code.py @@ -13,6 +13,7 @@ class ErrorCode(IntEnum): LLM_TIMEOUT = 1002 LLM_RATE_LIMIT = 1003 EMBEDDING_FAILED = 1004 + STT_STREAM_ERROR = 1005 # 流式 STT (WebSocket) 处理异常 # 简历相关错误 (2xxx) RESUME_NOT_FOUND = 2001 diff --git a/app/config.py b/app/config.py index d028436..a64fd68 100644 --- a/app/config.py +++ b/app/config.py @@ -128,6 +128,17 @@ class VoiceInterviewSettings(BaseSettings): max_wait_for_continuation_ms: int = 7000 ai_question_max_chars: int = 120 + # ---- 流式 STT (FunASR) ---- + # 主用模型: SenseVoice-Small（中文 CER 7.81%，CPU 17x 实时，多语种/情感标签） + funasr_model: str = "iic/SenseVoiceSmall" + funasr_device: str = "cpu" # cpu / cuda + funasr_quantize: bool = True # int8 量化（CPU 模式推荐） + funasr_hf_endpoint: str = "https://hf-mirror.com" + # WebSocket 流式分块 + streaming_stt_chunk_ms: int = 200 # 每帧时长 (ms) + streaming_stt_sample_rate: int = 16000 # PCM 采样率 (Hz) + streaming_stt_max_session_seconds: int = 600 # 单次会话硬上限 (s) + class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") diff --git a/app/main.py b/app/main.py index 3cd161d..931374e 100644 --- a/app/main.py +++ b/app/main.py @@ -214,6 +214,7 @@ def _register_routers(app: FastAPI) -> None: from app.modules.demo.router import router as demo_router from app.modules.interview.router import router as interview_router from app.modules.interview.skill_router import router as skill_router + from app.modules.interview.ws_router import router as interview_ws_router from app.modules.knowledge_base.cross_kb_router import router as cross_kb_router from app.modules.knowledge_base.rag_router import router as rag_router from app.modules.knowledge_base.router import router as kb_router @@ -229,6 +230,7 @@ def _register_routers(app: FastAPI) -> None: app.include_router(training_router, prefix="/api/training", tags=["个人训练计划"]) app.include_router(interview_router, prefix="/api/interview", tags=["模拟面试"]) app.include_router(skill_router, prefix="/api/interview/skills", tags=["面试方向"]) + app.include_router(interview_ws_router, prefix="/api/interview", tags=["面试 WebSocket"]) app.include_router(kb_router, prefix="/api/knowledgebase", tags=["知识库管理"]) app.include_router(rag_router, prefix="/api/knowledgebase", tags=["知识库问答"]) app.include_router(cross_kb_router, prefix="/api/cross-knowledgebase", tags=["跨知识库问答"]) diff --git a/app/modules/interview/voice_streaming_service.py b/app/modules/interview/voice_streaming_service.py new file mode 100644 index 0000000..e5abb28 --- /dev/null +++ b/app/modules/interview/voice_streaming_service.py @@ -0,0 +1,266 @@ +"""WebSocket 流式 STT 服务，基于 FunASR (SenseVoice-Small)。 + +设计要点： +- Lazy singleton + 线程安全双检锁：FunASR 模型 (~230MB) 只加载一次 +- 进程内运行：不开独立 funasr-server，与 FastAPI 同进程 +- 滚动窗口推理：每 1s 取最近 5s 音频做一次推理，partial 随音频增长而增长 +- asyncio.to_thread 跑阻塞推理，不阻塞事件循环 +- WS 下行事件用 dataclass + to_dict，WS 路由层直接 send_json + +为什么用滚动窗口而不是"每帧全量推理"？ +- SenseVoice 不是原生流式模型，全量推理是 O(n²) +- 滚动窗口把单次推理量固定在 ~5s 音频，CPU 可控 +- partial 文本随窗口滚动自然增长，体感"边说边出字" + +FunASR 输出格式： +- 返回 list[OrderedDict]，每个含 'text' / 'lang' / 'timestamp' 等 +- SenseVoice 在 text 前会带 <|zh|><|NEUTRAL|><|Speech|><|withitn|> 标签 +- _clean_sensevoice_output 把这些标签移除 +""" +from __future__ import annotations + +import asyncio +import logging +import re +import threading +import time +from collections.abc import AsyncIterator +from dataclasses import dataclass +from enum import Enum + +import numpy as np + +from app.common.error_code import ErrorCode +from app.config import settings + +logger = logging.getLogger(__name__) + + +class STTEventType(str, Enum): + """WebSocket 下行事件类型。""" + + PARTIAL = "partial" + FINAL = "final" + ERROR = "error" + + +@dataclass +class STTEvent: + """WebSocket 下行事件（partial / final / error）。""" + + type: STTEventType + text: str = "" + t0: float = 0.0 + t1: float = 0.0 + code: int = 0 + message: str = "" + + def to_dict(self) -> dict: + return { + "type": self.type.value, + "text": self.text, + "t0": self.t0, + "t1": self.t1, + "code": self.code, + "message": self.message, + } + + +class VoiceStreamingService: + """FunASR SenseVoice-Small 流式 STT 服务（lazy singleton，线程安全）。""" + + # 每次推理只取最近 5s 音频（控制单次推理量） + _ROLLING_WINDOW_SECONDS = 5 + # 每 1000ms 触发一次部分识别 + _INFER_INTERVAL_MS = 1000 + # 连续错误上限：超过则停止 stream 并报 error + _MAX_CONSECUTIVE_ERRORS = 3 + + _instance: "VoiceStreamingService | None" = None + _instance_lock = threading.Lock() + + def __new__(cls) -> "VoiceStreamingService": + if cls._instance is None: + with cls._instance_lock: + if cls._instance is None: + instance = super().__new__(cls) + instance._initialized = False + cls._instance = instance + return cls._instance + + def __init__(self) -> None: + if self._initialized: + return + self._model = None + self._model_lock = threading.Lock() + self._initialized = True + + def _get_model(self): + """Lazy load FunASR model。首次调用时下载并加载模型。""" + if self._model is None: + with self._model_lock: + if self._model is None: + logger.info( + "loading FunASR model: %s (device=%s, quantize=%s)", + settings.voice_interview.funasr_model, + settings.voice_interview.funasr_device, + settings.voice_interview.funasr_quantize, + ) + from funasr import AutoModel # 重量级延迟导入 + + self._model = AutoModel( + model=settings.voice_interview.funasr_model, + device=settings.voice_interview.funasr_device, + quantize=settings.voice_interview.funasr_quantize, + disable_update=True, + ) + logger.info("FunASR model loaded") + return self._model + + def _transcribe_sync(self, pcm_bytes: bytes, sample_rate: int) -> str: + """同步调用 FunASR。返回识别文本（已清理标签）。 + + 必须在 to_thread 中调用。 + """ + if not pcm_bytes: + return "" + # Int16 PCM (-32768..32767) → float32 (-1.0..1.0) + audio = np.frombuffer(pcm_bytes, dtype=np.int16).astype(np.float32) / 32768.0 + + model = self._get_model() + with self._model_lock: + result = model.generate( + input=audio, + sampling_rate=sample_rate, + disable_pbar=True, + ) + + if not result: + return "" + first = result[0] if isinstance(result, list) else result + text = first.get("text", "") if hasattr(first, "get") else "" + return _clean_sensevoice_output(text).strip() + + async def stream_transcribe( + self, + audio_chunks: AsyncIterator[bytes], + sample_rate: int | None = None, + ) -> AsyncIterator[STTEvent]: + """流式识别。 + + Args: + audio_chunks: 异步迭代器，每项是单声道 Int16 LE PCM bytes + sample_rate: 采样率，默认用配置值 (16kHz) + + Yields: + STTEvent 序列：多个 partial → 一个 final（或 error 终止） + """ + if sample_rate is None: + sample_rate = settings.voice_interview.streaming_stt_sample_rate + + bytes_per_second = sample_rate * 2 # Int16 = 2 bytes + rolling_window_bytes = self._ROLLING_WINDOW_SECONDS * bytes_per_second + max_session_seconds = settings.voice_interview.streaming_stt_max_session_seconds + + buffer = bytearray() + started_at = time.monotonic() + last_text = "" + last_infer_at = 0.0 + consecutive_errors = 0 + + try: + async for pcm in audio_chunks: + # 硬上限：超过 max_session_seconds 主动终止 + if time.monotonic() - started_at > max_session_seconds: + yield STTEvent( + type=STTEventType.ERROR, + code=ErrorCode.STT_STREAM_ERROR.value, + message="session_timeout", + ) + return + + buffer.extend(pcm) + now = time.monotonic() + + # 周期性 partial 推理 + if len(buffer) > 0 and (now - last_infer_at) * 1000 >= self._INFER_INTERVAL_MS: + last_infer_at = now + audio_chunk = ( + bytes(buffer[-rolling_window_bytes:]) + if len(buffer) > rolling_window_bytes + else bytes(buffer) + ) + try: + text = await asyncio.to_thread( + self._transcribe_sync, audio_chunk, sample_rate + ) + consecutive_errors = 0 + if text and text != last_text: + last_text = text + yield STTEvent( + type=STTEventType.PARTIAL, + text=text, + t0=0.0, + t1=now - started_at, + ) + except Exception as e: # noqa: BLE001 + consecutive_errors += 1 + logger.warning("FunASR partial failed (consecutive=%d): %s", consecutive_errors, e) + if consecutive_errors >= self._MAX_CONSECUTIVE_ERRORS: + yield STTEvent( + type=STTEventType.ERROR, + code=ErrorCode.STT_STREAM_ERROR.value, + message=f"too many consecutive errors: {e}", + ) + return + yield STTEvent( + type=STTEventType.ERROR, + code=ErrorCode.STT_STREAM_ERROR.value, + message=f"transcribe failed: {e}", + ) + except Exception as e: # noqa: BLE001 + logger.exception("stream_transcribe outer failure") + yield STTEvent( + type=STTEventType.ERROR, + code=ErrorCode.STT_STREAM_ERROR.value, + message=f"stream error: {e}", + ) + return + + # Final：流结束后对完整 buffer 跑一次推理 + # 即使 buffer 为空也发 final 事件（客户端用来确认流正常结束） + try: + text = "" + if buffer: + text = await asyncio.to_thread( + self._transcribe_sync, bytes(buffer), sample_rate + ) + yield STTEvent( + type=STTEventType.FINAL, + text=text, + t0=0.0, + t1=time.monotonic() - started_at, + ) + except Exception as e: # noqa: BLE001 + logger.exception("FunASR final failed") + yield STTEvent( + type=STTEventType.ERROR, + code=ErrorCode.STT_STREAM_ERROR.value, + message=f"finalize failed: {e}", + ) + + +_SENSEVOICE_TAG_RE = re.compile(r"<\|[^|]+\|>") + + +def _clean_sensevoice_output(text: str) -> str: + """清理 SenseVoice 输出的特殊标签。 + + SenseVoice 默认会在 text 前面带 <|lang|><|emotion|><|type|><|itn|> 等标签， + 例如 <|zh|><|NEUTRAL|><|Speech|><|withitn|>你好世界。流式场景下需要去掉。 + """ + return _SENSEVOICE_TAG_RE.sub("", text) + + +# 模块级单例 +voice_streaming_service = VoiceStreamingService() diff --git a/app/modules/interview/ws_router.py b/app/modules/interview/ws_router.py new file mode 100644 index 0000000..9d5eca0 --- /dev/null +++ b/app/modules/interview/ws_router.py @@ -0,0 +1,168 @@ +"""面试模块的 WebSocket 路由。 + +当前包含： +- /voice/stream: 流式 STT（FunASR SenseVoice-Small） + +WebSocket 鉴权说明： +- HTTP 的 auth_middleware 不覆盖 WebSocket，需要在 handler 内手动解码 JWT +- 鉴权方式：客户端通过 query 参数 ?token= 传 token +- 鉴权失败用 WS close code 1008 + +线协议（参考 voice_streaming_service.py 中的 STTEvent）： +- C→S 第一帧（可选）：JSON {"type":"start", "sampleRate":16000, "language":"zh"} +- C→S 后续帧：binary（Int16 LE 单声道 PCM，每帧 100-250ms） +- C→S 终止（可选）：JSON {"type":"end"} +- S→C：JSON {"type":"partial|final|error", ...} +""" +import asyncio +import json +import logging +from typing import AsyncIterator + +from fastapi import APIRouter, Query, WebSocket, WebSocketDisconnect, status + +from app.modules.auth.security import decode_access_token +from app.modules.interview.voice_streaming_service import STTEventType, voice_streaming_service + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +async def _pcm_chunker(websocket: WebSocket) -> AsyncIterator[bytes]: + """从 WebSocket 持续读取 binary 帧，遇到 end 帧 / disconnect 停止。""" + while True: + try: + msg = await websocket.receive() + except WebSocketDisconnect: + return + + msg_type = msg.get("type") + if msg_type == "websocket.disconnect": + return + + if "bytes" in msg and msg["bytes"] is not None: + yield msg["bytes"] + elif "text" in msg and msg["text"] is not None: + try: + payload = json.loads(msg["text"]) + except json.JSONDecodeError: + logger.warning("invalid JSON control frame: %r", msg["text"]) + continue + if payload.get("type") == "end": + return + # 其他类型（start 已经在 handler 里读过），忽略 + + +@router.websocket("/voice/stream") +async def voice_stream_ws( + websocket: WebSocket, + token: str = Query(..., description="JWT access token"), +) -> None: + """流式 STT WebSocket 端点。客户端必须带 ?token=。""" + # ---- 1. 鉴权（HTTP 中间件不覆盖 WS）---- + payload = decode_access_token(token) + if payload is None: + await websocket.close(code=status.WS_1008_POLICY_VIOLATION, reason="unauthorized") + return + try: + user_id = int(payload.get("sub")) + except (TypeError, ValueError): + await websocket.close(code=status.WS_1008_POLICY_VIOLATION, reason="invalid_token") + return + + await websocket.accept() + + # ---- 2. 尝试读首帧 ---- + # 可能是 start（采样率）、end（直接结束）、bytes（直接推 PCM） + sample_rate: int | None = None + first_bytes: bytes | None = None + end_immediately = False + try: + first = await asyncio.wait_for(websocket.receive(), timeout=2.0) + if "text" in first and first["text"] is not None: + try: + payload_json = json.loads(first["text"]) + ptype = payload_json.get("type") + if ptype == "start": + sr = payload_json.get("sampleRate") + if isinstance(sr, int) and sr > 0: + sample_rate = sr + elif ptype == "end": + end_immediately = True + except json.JSONDecodeError: + logger.warning("first frame not valid JSON, ignored") + elif "bytes" in first and first["bytes"] is not None: + first_bytes = first["bytes"] + except asyncio.TimeoutError: + # 客户端没在 2s 内发首帧 — 正常，继续 + pass + + # 空流：直接发 final 然后关闭 + if end_immediately: + try: + await websocket.send_json( + { + "type": "final", + "text": "", + "t0": 0.0, + "t1": 0.0, + "code": 0, + "message": "", + } + ) + except Exception: # noqa: BLE001 + pass + try: + await websocket.close(code=status.WS_1000_NORMAL_CLOSURE) + except Exception: # noqa: BLE001 + pass + return + + # ---- 3. 构造 PCM 流迭代器 ---- + if first_bytes is not None: + async def _chunks() -> AsyncIterator[bytes]: + yield first_bytes + async for c in _pcm_chunker(websocket): + yield c + chunks_iter: AsyncIterator[bytes] = _chunks() + else: + chunks_iter = _pcm_chunker(websocket) + + # ---- 4. 跑流式识别 ---- + try: + async for event in voice_streaming_service.stream_transcribe( + chunks_iter, sample_rate=sample_rate + ): + await websocket.send_json(event.to_dict()) + if event.type == STTEventType.FINAL: + break + except WebSocketDisconnect: + logger.info("voice stream WS disconnected by client (user=%s)", user_id) + return + except Exception as e: # noqa: BLE001 + logger.exception("voice stream WS failed (user=%s)", user_id) + try: + await websocket.send_json( + { + "type": "error", + "code": 1005, + "message": f"server error: {e}", + "text": "", + "t0": 0.0, + "t1": 0.0, + } + ) + except Exception: # noqa: BLE001 + pass + try: + await websocket.close(code=status.WS_1011_INTERNAL_ERROR) + except Exception: # noqa: BLE001 + pass + return + + # ---- 5. 正常关闭 ---- + try: + await websocket.close(code=status.WS_1000_NORMAL_CLOSURE) + except Exception: # noqa: BLE001 + pass diff --git a/docs/research/stt-selection-report.json b/docs/research/stt-selection-report.json new file mode 100644 index 0000000..a50709b --- /dev/null +++ b/docs/research/stt-selection-report.json @@ -0,0 +1,8 @@ +{ + "summary": "STT 选型深度研究 — 数字人面试场景 (faster-whisper 原理 + 4 类方案横评 + 集成路径)", + "agentCount": 93, + "logs": [ + "搜索: 51 条 finding | 去重后: 49 | 校验存活: 2 | 反驳: 23" + ], + "result": "# STT 选型深度报告：数字人模拟面试场景\n\n> 场景：浏览器录音 → STT → DeepSeek/dashscope LLM → TTS → 数字人开口\n> 目标：端到端延迟 < 1s（< 2s 可接受），中文为主，已有 faster-whisper small + CPU + int8 + 整段上传\n\n---\n\n## 一、faster-whisper 原理深入\n\n### 1.1 CTranslate2 加速内核\n\nfaster-whisper 的核心是 CTranslate2 —— 一个基于 Transformer 的 C++ 推理引擎。它通过算子融合、批处理化、int8/int16 量化、CPU SIMD（AVX/AVX-512/NEON）以及 CUDA fused kernel，把 Whisper 推理从 PyTorch eager 模式中解放出来。`SYSTRAN/faster-whisper` 官方仓库的 benchmark（Large-v2 + 13 min 音频 + RTX 3070 Ti 8GB + beam=5）显示：`openai/whisper fp16` 需 2m23s，`faster-whisper fp16` 1m03s，`faster-whisper int8` 59s，开启 `batch_size=8` 后 int8 仅 16s —— 相对原版约 9x 提速。在 Intel Core i7-12700K 8 线程 + 13 min 音频 + small 模型 + beam=5 上：`openai/whisper fp32` 6m58s vs `faster-whisper int8` 1m42s，差距约 4x；`batch_size=8` 进一步压到 51s（约 8x）。\n\n> 参考来源：[faster-whisper 官方仓库](https://github.com/SYSTRAN/faster-whisper) / [transcribe.py 源码](https://github.com/SYSTRAN/faster-whisper/blob/master/faster_whisper/transcribe.py)\n\n### 1.2 VAD 过滤（vad_filter）\n\nfaster-whisper 内置 silero-vad（轻量 ONNX VAD 模型），把长音频切成只含语音的段，跳过静音，既减少 token 数，又抑制模型在静音段产生的幻觉文本。可调三参：\n\n- `min_silence_duration_ms`：默认 2000ms（\"短停顿即分句\"会切碎自然语言；面试场景建议调小到 600–800ms）\n- `speech_pad_ms`：默认 400ms（每段语音两端补的静音 padding，避免截断）\n- `threshold`：默认 0.5（VAD 判为人声的概率阈值，嘈杂环境可上浮 0.55–0.65）\n\n> 来源（已校验）：[faster-whisper VAD 文档](https://github.com/SYSTRAN/faster-whisper#vad-filter)\n\n### 1.3 beam 与解码\n\n`faster-whisper` 默认 `beam_size=5`（与 OpenAI 原版一致），这也是 README benchmark 中的标准配置。`openai/whisper` 默认 `beam=1`（greedy），所以两者在\"同 beam\"下对比才公平。CTranslate2 在通用机器翻译任务上默认 `beam=2`、推荐 `beam=1` 以获取更高吞吐，但 Whisper 走的不是 CTranslate2 的 translation 接口，因此 beam=5 仍是 faster-whisper 的推荐值。\n\n> 来源：[CTranslate2 性能文档 4.8.1](https://opennmt.net/CTranslate2/performance.html) / [faster-whisper 源码](https://github.com/SYSTRAN/faster-whisper/blob/master/faster_whisper/transcribe.py)\n\n### 1.4 compute_type 量化选项\n\n已校验的事实（来自 `faster_whisper/transcribe.py` 源码枚举）：\n\n| 取值 | 适用 | 特点 |\n| --- | --- | --- |\n| `int8` | CPU 最快 | 内存与 VRAM 最低，WER 与 fp16 差距极小 |\n| `float16` | GPU 推荐 | 精度接近 fp32，吞吐优于 fp32 |\n| `float32` | CPU 精度上限 | 速度最慢，与 int8 比 WER 无明显改善，不推荐 |\n| `int8_float16` | GPU 混合 | 显存紧张时首选，保持 fp16 权重而 int8 激活 |\n\n### 1.5 模型规模\n\n`openai/whisper-large-v3` 参数量 1,543,490,560（约 1.54B），F16 safetensors 体积约 2.87 GB，训练数据 5M 小时（1M 弱标注 + 4M 伪标注），支持 99 种语言，mel 频谱 bin 数 128（v1/v2 为 80），相比 large-v2 在各语种上错误率下降 10–20%。\n\n> 来源：[openai/whisper-large-v3 HF 模型卡](https://huggingface.co/openai/whisper-large-v3) / [OpenAI Whisper Discussion #1762](https://github.com/openai/whisper/discussions/1762)\n\n### 1.6 CPU vs GPU 差距\n\n依据同源 benchmark（13 min 音频、small 模型、beam=5、int8）：\n- Intel i7-12700K（8 线程，CPU）：1m42s（`batch_size=1`）→ 51s（`batch_size=8`）\n- RTX 3070 Ti 8GB（GPU）：0m16s（`batch_size=8` int8）\n- 经验差距：单流场景 GPU 比 CPU 快约 3–5x（与具体模型/驱动相关）\n\n> 数字人面试是单流低并发，CPU 上的 small + int8 已可压到 < 1x 实时比；问题不在算力，而在\"整段上传\"导致的首字延迟。\n\n### 1.7 已知局限\n\n- 幻觉：社区报告显示 `large-v2` 在 45 min 音频上约 1–3 次幻觉，`large-v3` 默认阈值下约 5–10+ 次（logprob_threshold=-1, compression_ratio_threshold=2.4, no_speech_threshold=0.6）。\n- 整段上传导致首字延迟不可控：30s 音频必须等满 30s 才开始解码，对延迟敏感的对话是致命问题。\n- 默认 chunk_length 30s，whisper_streaming 之类的工具就是通过滑窗分块模拟\"流式\"。\n\n---\n\n## 二、全景横评\n\n### 表 1：本地推理类（基于 Whisper 家族）\n\n| 方案 | 原理 | 流式 | 中文 | WER | 延迟 | 成本 | 自托管 | 链接 |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| **faster-whisper** | CTranslate2 加速 Whisper | 仅整段（需 wrapper 模拟流式） | 中（large-v3） | large-v3 CV-zh 8–10% 区间 | small+CPU int8 ≈ 1.7x 实时 | 仅算力 | 是 | [github](https://github.com/SYSTRAN/faster-whisper) |\n| **whisper.cpp** | C/C++ + GGML，Apple Silicon Metal/CoreML | 原生 `--stream` | 中 | large-v3 FP16 接近 faster-whisper int8 | M1/M2 FP16 比 faster-whisper CPU 快 | 极低（本地） | 是 | [github](https://github.com/ggerganov/whisper.cpp) |\n| **WhisperX** | faster-whisper + wav2vec2 对齐 + pyannote 说话人分离 | 仅整段 | 中 | 与 large-v2/v3 持平 | RTF 慢 20–40% | 中 | 是 | [github](https://github.com/m-bain/whisperX) |\n| **Distil-Whisper** | Whisper large-v2/v3 蒸馏，decoder 32→2 层 | 仅整段 | 中（仅基于 Whisper 训练） | 短音频 ≤30s 持平 large-v2 | GPU 5–6x 提速 | 中 | 是 | [HF](https://huggingface.co/distil-whisper) |\n| **openai-whisper** | PyTorch eager 原版 | 不支持 | 中 | baseline | large-v3 fp16 RTF≈0.3，比 faster-whisper 慢 4–5x | 高 | 是 | [github](https://github.com/openai/whisper) |\n| **insanely-fast-whisper** | faster-whisper + flash-attn 2 + chunked long-form + batched | 可模拟流式 | 中 | 与 large-v3 持平 | 比 faster-whisper 额外 20–40% 吞吐 | 中 | 是 | [github](https://github.com/ Vaibhavs10/insanely-fast-whisper) |\n\n> 表中\"中文 WER\"列的具体数字因 benchmark 集差异较大（Common Voice 8 / Fleurs / WenetSpeech / AISHELL-1 测出来不一样），建议只做量级判断。\n\n### 表 2：中文优化类（达摩院 / WeNet 系）\n\n| 方案 | 原理 | 流式 | 中文 | WER/CER | 延迟 | 成本 | 自托管 | 链接 |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n| **Paraformer-Large** | 非自回归 Transformer + CIF predictor + 双向 decoder（220M） | 否（要 streaming 版） | 优 | AISHELL-1 CER 1.95%（带 LM）/ 2.85%（无 LM）；WenetSpeech test_meeting 14.59% | GPU RTF ≈ 0.0075（≈133x 实时）；CPU RTF ≈ 0.0005 量级（≈2000x 实时） | 低 | 是 | [FunASR](https://github.com/modelscope/FunASR) |\n| **SeACo-Paraformer** | Paraformer + contextual bias 热词模块 | 否 | 优 | 垂类热词 CER/F1 改善 10–30% | 同 Paraformer | 低 | 是 | [FunASR](https://github.com/modelscope/FunASR) |\n| **Streaming-Paraformer** | chunk-based 编码 | 是 | 优 | 同 Paraformer | **首字 < 300ms** | 低 | 是 | [FunASR](https://github.com/modelscope/FunASR) |\n| **SenseVoice-Small** | 234M 多语种 ASR+语种+情感 | 是 | 优（含粤/吴/闽等方言） | 中文 AISHELL-1/2 优于 Whisper-large-v3 | CPU 17x 实时 / GPU 170x 实时；约 Whisper-Large-v3 的 15x | 低 | 是 | [FunASR](https://github.com/modelscope/FunASR) |\n| **WeNet Conformer U2++** | U2++ 端到端 Conformer，runtime int8 | 是 | 优 | LibriSpeech test-clean 2.45–3.09% | RTF 依硬件而定，int8 量化可下沉 | 低 | 是 | [WeNet](https://github.com/wenet-e2e/wenet) |\n\n> 中文 184 音频 / 192 分钟 benchmark（CER）：**SenseVoice-Small 7.81%**、**Fun-ASR-Nano 8.20%**、**Paraformer-Large 10.18%**、**Whisper-large-v3-turbo 21.71%**、**Whisper-large-v3 20.02%** —— 中文场景下 Paraformer/SenseVoice 相对 Whisper 有数量级优势。\n\n### 表 3：云服务类\n\n| 方案 | 原理 | 流式 | 中文 | 准确率/延迟 | 成本 | 自托管 | 链接 |\n| --- | --- | --- | --- | --- | --- | --- | --- |\n| **阿里云智能语音交互（ISI）** | 阿里云 ASR 全家桶 | 是（实时语音识别） | 优 | 商用 SLA 强；延迟/准确率官网未公开 | ¥0.04/分钟 起；新用户 3 个月免费（2 路并发、2 小时/天） | 否 | [阿里云](https://help.aliyun.com/zh/isi/getting-started/start-with-quick-start) |\n| **讯飞实时语音听写** | 短音频 ≤60s | 是 | 优（含 23 种方言） | 宣称 98% 识别率；1 小时音频 ≈ 20 秒转写 | 新用户 500 次/天；一次性 5 万次/90 天；升级 100 万次 | 否 | [讯飞](https://www.xfyun.cn/services/voicedictation) |\n\n### 表 4：流式/实时类（专门面向\"边说边出字\"）\n\n| 方案 | 原理 | 流式 | 中文 | 延迟 | 成本 | 自托管 | 链接 |\n| --- | --- | --- | --- | --- | --- | --- | --- |\n| **whisper_streaming**（ufal） | 滑窗 + LocalAgreement (n=2) 模拟流式 | 是（按 1s chunk 增量） | 中（与底层 Whisper 一致） | 论文 headline 3.3s 端到端；chunk 大小自适应 | 仅算力 | 是 | [github](https://github.com/ufal/whisper_streaming) |\n| **Streaming-Paraformer** | chunk-based 编码 + 流式解码 | 是 | 优 | **首字 < 300ms** | 仅算力 | 是 | [FunASR](https://github.com/modelscope/FunASR) |\n| **SenseVoice-Small** | 单次前向 + chunked inference | 是 | 优 | CPU 17x 实时可近流式 | 仅算力 | 是 | [FunASR](https://github.com/modelscope/FunASR) |\n| **FunASR Server** | `funasr-server` CLI（v1.3.3+） | 是 | 优 | 集成 VAD + ASR + 标点 | 仅算力 | 是 | [FunASR](https://github.com/modelscope/FunASR) |\n| **阿里云实时语音识别 / 讯飞实时听写** | 云端流式 | 是 | 优 | 商用 SLA | 按分钟计费 | 否 | 见表 3 |\n\n---\n\n## 三、数字人面试选型建议\n\n### 3.1 流式 STT 推荐\n\n**首选：FunASR Streaming-Paraformer + SenseVoice-Small 二选一**\n\n| 候选 | 选它的场景 |\n| --- | --- |\n| **Streaming-Paraformer** | 需要硬 < 1s 端到端、首字 < 300ms；只识别中文 |\n| **SenseVoice-Small** | 需要语种/方言/情感标签；多语种混合输入（粤语、吴语） |\n\n**核心理由**：\n1. 中文 CER 显著低于 Whisper（184-音频中文 benchmark：SenseVoice 7.81% vs Whisper-large-v3 20.02%）。\n2. CPU 上 17x 实时比（SenseVoice-Small），单核即可支撑面试并发，**不需要 GPU**。\n3. 首字延迟 < 300ms，叠加 LLM + TTS 后端到端仍可压到 1s 内。\n4. 商业友好（MIT/Apache，模型权重可商用）。\n\n### 3.2 中文准确率最高方案\n\n**SeACo-Paraformer-large + 热词表**：\n- 加载面试领域热词（编程语言名、框架名、面试专有名词）可在不重训模型下把 CER 改善 10–30%。\n- 准确率兜底：当流式输出置信度低或长句被切碎时，回退到 SeACo-Paraformer 做整段二次校对。\n\n### 3.3 延迟 vs 准确率 权衡\n\n| 优先级 | 推荐组合 | 端到端预期 |\n| --- | --- | --- |\n| **极致延迟（< 1s）** | Streaming-Paraformer / SenseVoice 流式 → DeepSeek → TTS | 800–1000ms |\n| **准确率优先** | faster-whisper large-v3 int8（CPU/GPU 整段）→ DeepSeek → TTS | 2–4s（受 30s chunk 限制） |\n| **均衡** | SenseVoice 流式（首字 < 300ms）+ SeACo-Paraformer 兜底 | 1–1.5s |\n\n> 当前 faster-whisper small + CPU + int8 + 整段上传的瓶颈**不在算力，而在\"整段上传\"**：用户每说 30s 才能拿到一次识别结果，体感是 5–10s 一次响应。**改成流式是首字延迟从\"一句话\"压到\"一个词\"的关键。**\n\n### 3.4 推荐组合（流式 + 兜底 + 长音频）\n\n```\n┌─────────────────────────────────────────────┐\n│ 主用（流式）：SenseVoice-Small / Streaming- │\n│ Paraformer，1s chunk 增量输出 │\n├─────────────────────────────────────────────┤\n│ 兜底（高准确率）：SeACo-Paraformer-large │\n│ + 热词表；用户停顿时触发整段二次校对 │\n├─────────────────────────────────────────────┤\n│ 长音频兜底：whisper_streaming (large-v3) │\n│ 会议/复盘场景 30s+ 语音 │\n└─────────────────────────────────────────────┘\n```\n\n### 3.5 成本估算（自托管 GPU vs 云 API）\n\n**自托管 CPU + FunASR**（推荐，面试并发 < 100）：\n- 硬件：单台 16 核 CPU 服务器 ≈ ¥3,000/月（云厂商按量）\n- 单流：SenseVoice CPU 17x 实时 → 单核可支撑 1 路面试\n- 16 核可同时撑 16 路面试\n- 单次面试 30min STT 算力成本 < ¥0.01\n\n**自托管 GPU + Whisper 家族**：\n- 硬件：1× A10/A100 ≈ ¥4,000–8,000/月\n- Whisper large-v3 int8 吞吐：约 8x 实时（13 min / 1m42s 推算），单卡可撑 8 路\n- 单次面试 30min STT 算力成本 ≈ ¥0.05–0.10\n\n**云 API（按分钟计费）**：\n- 阿里云 ¥0.04/分钟 → 30min 面试 = ¥1.2\n- 讯飞 500 次/天免费，超出按量计费\n- 1000 次面试/月 ≈ ¥1,200（仅 STT 成本）\n\n**结论**：自托管 FunASR 是当前场景的成本与延迟最优解。\n\n---\n\n## 四、端到端集成路径\n\n### 4.1 浏览器端采集\n\n```\n[AudioWorklet] → 16kHz mono PCM Int16 → [WebSocket binary]\n```\n\n- 采样率：16 kHz（Whisper / Paraformer / SenseVoice 训练分辨率）\n- 编码：原始 PCM Int16 Little-Endian（避免 WAV 头开销）；或 Opus 16kbps 节省带宽\n- 缓冲：每 100ms 推一帧 PCM\n- VAD：浏览器端可选 `@ricky0123/vad-web`（silero-vad 移植）做端点检测，避免空包\n\n### 4.2 后端 WebSocket 服务\n\n**方案 A：FastAPI WebSocket + SenseVoice-Small（CPU）**\n```python\n# 伪代码\n@app.websocket(\"/ws/asr\")\nasync def asr(ws: WebSocket):\n await ws.accept()\n model = load_sensevoice() # 单例\n pcm_buf = bytearray()\n async for frame in ws.iter_bytes():\n pcm_buf.extend(frame)\n # 每 1s 触发一次增量识别\n if len(pcm_buf) >= 16000 * 2: # 1s PCM\n chunk = bytes(pcm_buf)\n pcm_buf.clear()\n text = model.streaming_infer(chunk)\n await ws.send_json({\"partial\": text})\n```\n\n**方案 B：FunASR Server（v1.3.3+）**：\n```bash\nfunasr-server --model iic/SenseVoiceSmall --device cpu --port 10095\n```\nWebSocket 客户端按 1s chunk 推送 PCM，服务端按 VAD + chunk 输出 partial/final。\n\n### 4.3 端到端架构图\n\n```\n┌──────────┐ PCM/Opus ┌────────────┐ partial text ┌──────────┐\n│ Browser │ ──────────► │ FunASR │ ──────────────► │ DeepSeek │\n│ AudioWL │ │ WS Server │ │ LLM │\n└──────────┘ └────────────┘ └────┬─────┘\n ▲ │ reply\n │ ▼\n │ 数字人嘴型 + TTS 音频 ┌────────────┐ text│\n │ ◄─────────────────────────── │ TTS (Cosy) │ ◄──────────┘\n │ Voice │\n └────────────┘\n```\n\n### 4.4 现有 faster-whisper 升级到流式的步骤\n\n1. **保留现有 faster-whisper 作为兜底**（用 `vad_filter=True` + `vad_parameters={\"min_silence_duration_ms\": 600, \"speech_pad_ms\": 200}`，见 1.2 节已校验参数）。\n2. **新增流式通道**：部署 `funasr-server`（SenseVoice-Small 或 Streaming-Paraformer），监听独立端口。\n3. **前端改造**：用 AudioWorklet 替代一次性 MediaRecorder，按 1s chunk 推 WS。\n4. **灰度切流**：先 10% 流量走 FunASR 流式，对比首字延迟与 CER；稳定后全量切换。\n5. **热词加载**：SeACo-Paraformer 启动时注入面试领域热词 JSON（编程语言、框架、技术术语）。\n6. **延迟观测**：前端打点 `t_audio_end → t_first_partial → t_llm_first_token → t_tts_first_audio`，端到端 P95 目标 < 1.5s。\n\n---\n\n## 五、风险与备选\n\n| 风险 | 触发条件 | 备选方案 |\n| --- | --- | --- |\n| SenseVoice 对专有名词识别不稳 | 面试者提到冷门技术术语 | 切到 SeACo-Paraformer + 热词，或 Whisper large-v3 整段兜底 |\n| 浏览器麦克风权限被拒 | 用户未授权 | 引导降级到文本输入 |\n| WebSocket 断线 | 网络抖动 | 前端 AudioWorklet 缓冲 + 自动重连，丢帧可接受 |\n| 多人/重叠语音 | 多人面试 | 加 pyannote-audio 分离，或改为单工（按说话人切换） |\n| CPU 突发拥塞 | 高并发（> 16 路） | 横向扩容 FunASR worker，或临时切到阿里云 ISI |\n| 幻觉文本污染 LLM 输入 | VAD 漏检/麦克风噪声 | 加 `no_speech_threshold` / `logprob_threshold` 过滤低置信片段 |\n| 模型升级 breaking change | FunASR v2 大版本 | 锁版本 + 离线测试集回归 |\n| 监管合规（录音存证） | 需要审计 | 同步落库原始 PCM + ASR 中间结果（MinIO/PG） |\n\n**Plan B（云服务兜底）**：自托管 FunASR 故障时，前端 5xx 自动重试到阿里云 ISI（¥0.04/min），SLA 由云厂商兜底。\n\n---\n\n## 六、引用来源\n\n### faster-whisper / CTranslate2\n- [SYSTRAN/faster-whisper GitHub](https://github.com/SYSTRAN/faster-whisper) — 24,266 stars，MIT，4x 提速声明\n- [faster_whisper/transcribe.py 源码](https://github.com/SYSTRAN/faster-whisper/blob/master/faster_whisper/transcribe.py) — compute_type 枚举 + benchmark 数据\n- [CTranslate2 Performance tips 4.8.1](https://opennmt.net/CTranslate2/performance.html) — beam_size 推荐\n- [faster-whisper VAD 文档](https://github.com/SYSTRAN/faster-whisper#vad-filter) — min_silence / speech_pad / threshold 默认值（已校验）\n\n### Whisper 模型卡与社区\n- [openai/whisper-large-v3 HF](https://huggingface.co/openai/whisper-large-v3) — 1.54B 参数、2.87GB F16、5M 训练时长、99 语种\n- [openai/whisper Discussion #1762](https://github.com/openai/whisper/discussions/1762) — large-v3 发布与幻觉讨论\n\n### FunASR（达摩院）\n- [modelscope/FunASR GitHub](https://github.com/modelscope/FunASR) — Paraformer / SeACo / SenseVoice / v1.3.3\n- [FunASR README_zh](https://github.com/modelscope/FunASR/blob/main/README_zh.md) — 184-音频中文 CER benchmark、RTF 数字\n\n### 流式 / 对齐\n- [ufal/whisper_streaming](https://github.com/ufal/whisper_streaming) — 3.3s 端到端、1s chunk、LocalAgreement n=2\n- [WhisperX](https://github.com/m-bain/whisperX) — wav2vec2 + pyannote 管线\n\n### WeNet\n- [wenet-e2e/wenet](https://github.com/wenet-e2e/wenet) — U2++ Conformer、LibriSpeech WER、int8 runtime\n\n### 云服务\n- [阿里云智能语音交互 快速入门](https://help.aliyun.com/zh/isi/getting-started/start-with-quick-start) — 免费试用 3 个月 / 2 路 / 2h/天\n- [讯飞实时语音听写](https://www.xfyun.cn/services/voicedictation) — ≤60s、98%、500 次/天免费\n\n---\n\n**核心结论（一句话）**：把 `faster-whisper small + CPU + int8 + 整段上传` 升级为 `FunASR Streaming-Paraformer / SenseVoice-Small + WebSocket 流式分块`，是当前架构下让\"数字人面试\"端到端 < 1s 的最高 ROI 改动；中文 CER 同步从 Whisper 的 20% 区间压到 8% 区间，准确率与延迟双收。" +} \ No newline at end of file diff --git a/frontend/public/audio-worklets/pcm-capture.js b/frontend/public/audio-worklets/pcm-capture.js new file mode 100644 index 0000000..3f08cad --- /dev/null +++ b/frontend/public/audio-worklets/pcm-capture.js @@ -0,0 +1,35 @@ +/** + * AudioWorklet: 浏览器音频采集处理器。 + * + * 在音频线程运行，接收输入设备的 Float32 帧（通常是 128 samples/帧）， + * 复制后通过 port.postMessage 转发到主线程。主线程负责： + * 1. Float32 → Int16 转换 + * 2. 48kHz → 16kHz 重采样（线性插值） + * 3. 按 200ms 切片 + * 4. 通过 WebSocket 推送到后端 + * + * 这个 worklet 不做重采样是因为： + * - AudioContext 的 sampleRate 跟设备/浏览器有关，48kHz 是常见值但不是固定 + * - 重采样逻辑放在主线程里更易调试 / 单测 + * - worklet 只负责"把原始帧搬出音频线程" + * + * 注意：这个文件是浏览器原生 JS（不是 TS），由 Vite 通过 public/ 目录 + * 直接以原文件形式 serve，URL 是 /audio-worklets/pcm-capture.js。 + */ + +class PCMCaptureProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (!input || input.length === 0) return true; + + const channel = input[0]; + if (!channel || channel.length === 0) return true; + + // 必须复制 — input 缓冲会被下一帧覆盖 + const copy = new Float32Array(channel); + this.port.postMessage(copy, [copy.buffer]); + return true; + } +} + +registerProcessor('pcm-capture', PCMCaptureProcessor); diff --git a/frontend/src/api/voiceStream.ts b/frontend/src/api/voiceStream.ts new file mode 100644 index 0000000..922f159 --- /dev/null +++ b/frontend/src/api/voiceStream.ts @@ -0,0 +1,136 @@ +/** + * VoiceStreamClient — 浏览器侧流式 STT WebSocket 客户端。 + * + * 用法： + * const client = new VoiceStreamClient({ onPartial, onFinal, onError }); + * await client.connect(accessToken, { sampleRate: 16000 }); + * client.sendAudio(pcmBytes); // 多次 + * client.endStream(); // 触发 server final + * // 或 client.close() 强制断开 + * + * 鉴权：token 通过 query 参数 ?token= 传给后端（HTTP 中间件不覆盖 WS） + * URL：开发走 Vite 代理 /api/interview/voice/stream，生产同源 + * + * 不做自动重连 — 上层 hook 决定降级到 batch 模式还是提示用户 + */ + +import { apiUrl } from './request'; +import type { STTEvent, STTStartMessage, STTEndMessage, VoiceStreamListeners } from '../types/voiceStream'; + +const WS_PATH = '/api/interview/voice/stream'; + +export interface VoiceStreamConfig { + sampleRate: number; + language?: string; + /** WS endpoint 路径，默认 /api/interview/voice/stream */ + path?: string; +} + +export class VoiceStreamClient { + private ws: WebSocket | null = null; + private listeners: VoiceStreamListeners; + + constructor(listeners: VoiceStreamListeners = {}) { + this.listeners = listeners; + } + + /** 打开 WS 连接。返回 Promise，连接成功 resolve，失败 reject。 */ + connect(token: string, config: VoiceStreamConfig): Promise { + return new Promise((resolve, reject) => { + const path = config.path ?? WS_PATH; + const url = `${apiUrl(path)}?token=${encodeURIComponent(token)}`; + const ws = new WebSocket(url); + // 注意：浏览器 WebSocket API 不支持设置 header，token 只能走 query + this.ws = ws; + + const onOpen = () => { + ws.removeEventListener('open', onOpen); + ws.removeEventListener('error', onError); + this.listeners.onOpen?.(); + // 立即发 start + const start: STTStartMessage = { + type: 'start', + sampleRate: config.sampleRate, + language: config.language, + }; + ws.send(JSON.stringify(start)); + resolve(); + }; + const onError = (ev: Event) => { + ws.removeEventListener('open', onOpen); + ws.removeEventListener('error', onError); + reject(new Error(`WebSocket connection failed: ${(ev as ErrorEvent).message ?? 'unknown'}`)); + }; + + ws.addEventListener('open', onOpen); + ws.addEventListener('error', onError); + ws.addEventListener('message', this.handleMessage); + ws.addEventListener('close', this.handleClose); + }); + } + + /** 发送一段 Int16 LE PCM bytes。WS 必须已 open。 */ + sendAudio(pcm: ArrayBuffer | Uint8Array): void { + if (!this.ws || this.ws.readyState !== WebSocket.OPEN) { + throw new Error('WebSocket is not open'); + } + // WebSocket.send 支持 Blob / ArrayBuffer / string + this.ws.send(pcm); + } + + /** 发送 end 控制帧，触发服务端 final。 */ + endStream(): void { + if (!this.ws || this.ws.readyState !== WebSocket.OPEN) { + return; + } + const end: STTEndMessage = { type: 'end' }; + this.ws.send(JSON.stringify(end)); + } + + /** 强制关闭 WS。 */ + close(code = 1000, reason = 'client_close'): void { + if (this.ws) { + try { + this.ws.close(code, reason); + } catch { + // ignore + } + this.ws = null; + } + } + + /** 是否已 open。 */ + isOpen(): boolean { + return this.ws?.readyState === WebSocket.OPEN; + } + + private handleMessage = (ev: MessageEvent) => { + if (typeof ev.data !== 'string') { + // 二进制帧不是协议的一部分 + return; + } + let event: STTEvent; + try { + event = JSON.parse(ev.data); + } catch { + this.listeners.onError?.(0, `Invalid JSON from server: ${ev.data.slice(0, 200)}`); + return; + } + switch (event.type) { + case 'partial': + this.listeners.onPartial?.(event.text, event.t1); + break; + case 'final': + this.listeners.onFinal?.(event.text, event.t1); + break; + case 'error': + this.listeners.onError?.(event.code, event.message); + break; + } + }; + + private handleClose = (ev: CloseEvent) => { + this.listeners.onClose?.(ev.code, ev.reason); + this.ws = null; + }; +} diff --git a/frontend/src/components/VoiceMicButton.tsx b/frontend/src/components/VoiceMicButton.tsx new file mode 100644 index 0000000..9d982b1 --- /dev/null +++ b/frontend/src/components/VoiceMicButton.tsx @@ -0,0 +1,74 @@ +/** + * VoiceMicButton — 麦克风按钮，5 态视觉。 + * + * idle (灰) - Mic 图标 + * recording (红) - MicOff 图标（录音中） + * streaming (蓝) - MicOff 图标 + 脉冲 + * transcribing (黄) - Loader2 旋转 + * error (红) - AlertCircle + * + * 颜色用 Tailwind class，hover/focus 跟周围环境保持一致。 + */ + +import { AlertCircle, Loader2, Mic, MicOff } from 'lucide-react'; +import type { VoiceState } from '../hooks/useVoiceInput'; + +interface VoiceMicButtonProps { + voiceState: VoiceState; + onClick: () => void; + disabled?: boolean; +} + +const STATE_CONFIG: Record = { + idle: { + icon: Mic, + className: 'bg-slate-100 text-slate-600 hover:bg-slate-200', + title: '点击开始录音', + }, + recording: { + icon: MicOff, + className: 'bg-red-500 text-white hover:bg-red-600 animate-pulse', + title: '点击停止（整段转写）', + }, + streaming: { + icon: MicOff, + className: 'bg-blue-500 text-white hover:bg-blue-600 animate-pulse', + title: '点击停止（流式识别）', + }, + transcribing: { + icon: Loader2, + className: 'bg-amber-100 text-amber-700 cursor-wait', + title: '正在转写...', + }, + error: { + icon: AlertCircle, + className: 'bg-red-100 text-red-600', + title: '录音出错', + }, +}; + +export function VoiceMicButton({ voiceState, onClick, disabled }: VoiceMicButtonProps) { + const config = STATE_CONFIG[voiceState]; + const Icon = config.icon; + const isSpinning = voiceState === 'transcribing'; + + return ( + + ); +} diff --git a/frontend/src/components/VoiceStatusLine.tsx b/frontend/src/components/VoiceStatusLine.tsx new file mode 100644 index 0000000..150df36 --- /dev/null +++ b/frontend/src/components/VoiceStatusLine.tsx @@ -0,0 +1,82 @@ +/** + * VoiceStatusLine — 语音状态行。 + * + * 显示当前 voice 状态 + 时长 / partial 预览 / 错误信息。 + * idle 状态下不渲染（节省空间）。 + * + * 样式紧凑：放在 textarea 旁边或下面。 + */ + +import { AlertCircle, Loader2, Mic, Radio } from 'lucide-react'; +import type { VoiceState } from '../hooks/useVoiceInput'; + +interface VoiceStatusLineProps { + voiceState: VoiceState; + recordingSeconds?: number; + partialText?: string; + voiceError?: string; +} + +function formatSeconds(s: number): string { + const m = Math.floor(s / 60); + const r = s % 60; + return `${m}:${r.toString().padStart(2, '0')}`; +} + +export function VoiceStatusLine({ + voiceState, + recordingSeconds = 0, + partialText = '', + voiceError = '', +}: VoiceStatusLineProps) { + if (voiceState === 'idle') { + return null; + } + + let Icon: typeof Mic; + let text: string; + let className: string; + + switch (voiceState) { + case 'recording': + Icon = Mic; + text = `录音中 ${formatSeconds(recordingSeconds)}`; + className = 'text-red-600'; + break; + case 'streaming': + Icon = Radio; + text = partialText + ? `正在识别：${partialText}` + : `正在识别 ${formatSeconds(recordingSeconds)}`; + className = 'text-blue-600'; + break; + case 'transcribing': + Icon = Loader2; + text = '正在转写...'; + className = 'text-amber-600'; + break; + case 'error': + Icon = AlertCircle; + text = voiceError || '录音出错'; + className = 'text-red-600'; + break; + default: + Icon = Mic; + text = ''; + className = ''; + } + + const isSpinning = voiceState === 'transcribing'; + + return ( +
+ + {text} +
+ ); +} diff --git a/frontend/src/hooks/useVoiceInput.ts b/frontend/src/hooks/useVoiceInput.ts new file mode 100644 index 0000000..1b07b42 --- /dev/null +++ b/frontend/src/hooks/useVoiceInput.ts @@ -0,0 +1,388 @@ +/** + * useVoiceInput — 统一的语音输入 hook。 + * + * mode='stream' (主用)： + * - getUserMedia + AudioContext + AudioWorklet 采集 PCM + * - 48kHz → 16kHz 抽取 + Float32 → Int16 转换 + * - 按 200ms 切片 → WebSocket 推送 → 收 partial / final + * - onCommit 在收到 final 时调 + * + * mode='batch' (回退)： + * - getUserMedia + MediaRecorder 采集到 Blob + * - stop 时调 interviewApi.transcribeVoice → onCommit(text) + * + * 状态机：idle → recording/streaming → transcribing（仅 batch） → idle + * ↓ + * error → 1.5s 后回 idle + * + * 不做自动重连 — 流式失败时上层可切到 batch 模式重试 + */ + +import { useCallback, useEffect, useRef, useState } from 'react'; +import { interviewApi } from '../api/interview'; +import { VoiceStreamClient } from '../api/voiceStream'; +import { + PCM_SAMPLE_RATE, + VOICE_ERROR_MESSAGES, + cleanTranscript, + getAudioFileExtension, + getPreferredAudioMimeType, +} from '../utils/voice'; + +export type VoiceMode = 'batch' | 'stream'; +export type VoiceState = 'idle' | 'recording' | 'streaming' | 'transcribing' | 'error'; + +export interface UseVoiceInputOptions { + mode: VoiceMode; + /** 用户接受结果时（流式 final / 批式 transcribe 完成）调 */ + onCommit: (text: string) => void; + /** 错误回调（鉴权失败、ASR 错误、设备无权限等） */ + onError?: (err: Error) => void; + /** 拿当前 JWT（WS 鉴权用），返回 null 表示未登录 */ + getToken?: () => string | null; +} + +export interface UseVoiceInputReturn { + voiceState: VoiceState; + voiceError: string; + recordingSeconds: number; + partialText: string; + start: () => Promise; + stop: () => Promise; + cancel: () => void; +} + +const ERROR_RECOVER_MS = 1500; +const STREAM_CHUNK_MS = 200; // 每 200ms 推一段 + +function getVoiceErrorMessage(err: unknown): string { + if (err instanceof Error) { + if (err.name in VOICE_ERROR_MESSAGES) { + return VOICE_ERROR_MESSAGES[err.name] ?? err.message; + } + return err.message; + } + return '录音发生未知错误'; +} + +export function useVoiceInput(options: UseVoiceInputOptions): UseVoiceInputReturn { + const { mode, onCommit, onError, getToken } = options; + + const [voiceState, setVoiceState] = useState('idle'); + const [voiceError, setVoiceError] = useState(''); + const [recordingSeconds, setRecordingSeconds] = useState(0); + const [partialText, setPartialText] = useState(''); + + // Refs（不变更时不需要触发渲染） + const mediaStreamRef = useRef(null); + const mediaRecorderRef = useRef(null); + const audioChunksRef = useRef([]); + const recordingTimerRef = useRef(null); + const errorRecoverTimerRef = useRef(null); + + // 流式专用 + const audioContextRef = useRef(null); + const workletNodeRef = useRef(null); + const sourceNodeRef = useRef(null); + const wsClientRef = useRef(null); + // PCM 缓冲（Int16 at 16kHz） + const pcmBufferRef = useRef([]); + const sourceSampleRateRef = useRef(PCM_SAMPLE_RATE); + + // 稳定的 callback refs（避免 useEffect 反复重连） + const onCommitRef = useRef(onCommit); + const onErrorRef = useRef(onError); + onCommitRef.current = onCommit; + onErrorRef.current = onError; + + const enterError = useCallback((err: unknown) => { + const msg = getVoiceErrorMessage(err); + setVoiceError(msg); + setVoiceState('error'); + onErrorRef.current?.(err instanceof Error ? err : new Error(msg)); + // 1.5s 后自动回 idle + if (errorRecoverTimerRef.current !== null) { + window.clearTimeout(errorRecoverTimerRef.current); + } + errorRecoverTimerRef.current = window.setTimeout(() => { + setVoiceState('idle'); + setVoiceError(''); + setPartialText(''); + }, ERROR_RECOVER_MS); + }, []); + + const startTimer = useCallback(() => { + if (recordingTimerRef.current !== null) return; + const startTime = Date.now(); + setRecordingSeconds(0); + recordingTimerRef.current = window.setInterval(() => { + setRecordingSeconds(Math.floor((Date.now() - startTime) / 1000)); + }, 1000); + }, []); + + const stopTimer = useCallback(() => { + if (recordingTimerRef.current !== null) { + window.clearInterval(recordingTimerRef.current); + recordingTimerRef.current = null; + } + }, []); + + // ============================ 流式 ============================ + const startStream = useCallback(async () => { + setVoiceError(''); + setPartialText(''); + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + mediaStreamRef.current = stream; + + // 准备 WebSocket + const token = getToken?.(); + if (!token) { + stream.getTracks().forEach((t) => t.stop()); + mediaStreamRef.current = null; + throw new Error('未登录，无法使用流式识别'); + } + + const wsClient = new VoiceStreamClient({ + onPartial: (text) => { + setPartialText(cleanTranscript(text)); + }, + onFinal: (text) => { + const cleaned = cleanTranscript(text); + if (cleaned) onCommitRef.current(cleaned); + setVoiceState('idle'); + setPartialText(''); + }, + onError: (code, message) => { + enterError(new Error(`STT error (${code}): ${message}`)); + }, + onClose: () => { + setVoiceState((prev) => (prev === 'streaming' ? 'idle' : prev)); + }, + }); + wsClientRef.current = wsClient; + + // 准备 AudioContext + AudioWorklet + const AudioCtxCtor: typeof AudioContext = window.AudioContext + || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext; + const ctx = new AudioCtxCtor(); + audioContextRef.current = ctx; + sourceSampleRateRef.current = ctx.sampleRate; + + await ctx.audioWorklet.addModule('/audio-worklets/pcm-capture.js'); + const workletNode = new AudioWorkletNode(ctx, 'pcm-capture'); + workletNodeRef.current = workletNode; + + const sourceNode = ctx.createMediaStreamSource(stream); + sourceNodeRef.current = sourceNode; + sourceNode.connect(workletNode); + + // Float32 帧 → Int16 抽取 → 缓冲到 200ms → WS 推 + const targetChunkSamples = Math.floor((STREAM_CHUNK_MS * PCM_SAMPLE_RATE) / 1000); + workletNode.port.onmessage = (ev) => { + const frame = ev.data as Float32Array; + // 抽样到 PCM_SAMPLE_RATE（默认源 48kHz 抽 3 倍） + const factor = Math.max(1, Math.round(sourceSampleRateRef.current / PCM_SAMPLE_RATE)); + for (let i = 0; i < frame.length; i += factor) { + const s = Math.max(-1, Math.min(1, frame[i])); + pcmBufferRef.current.push(s < 0 ? s * 0x8000 : s * 0x7fff); + } + if (pcmBufferRef.current.length >= targetChunkSamples) { + const chunk = new Int16Array(pcmBufferRef.current.splice(0, targetChunkSamples)); + try { + wsClientRef.current?.sendAudio(chunk.buffer); + } catch { + // WS 可能在重连中 + } + } + }; + + // 连接 WS（onopen 后自动发 start） + await wsClient.connect(token, { + sampleRate: PCM_SAMPLE_RATE, + language: 'zh', + }); + + setVoiceState('streaming'); + startTimer(); + }, [enterError, getToken, startTimer]); + + const stopStream = useCallback(async () => { + stopTimer(); + const ws = wsClientRef.current; + if (ws && ws.isOpen()) { + ws.endStream(); + } + // 等待 final 由 onFinal callback 触发，状态转换在 onFinal 里 + }, [stopTimer]); + + // ============================ 批式 ============================ + const startBatch = useCallback(async () => { + setVoiceError(''); + setPartialText(''); + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + mediaStreamRef.current = stream; + + const mimeType = getPreferredAudioMimeType(); + const recorder = new MediaRecorder( + stream, + mimeType ? { mimeType } : undefined, + ); + mediaRecorderRef.current = recorder; + audioChunksRef.current = []; + + recorder.ondataavailable = (event) => { + if (event.data.size > 0) { + audioChunksRef.current.push(event.data); + } + }; + + recorder.onstop = async () => { + stopTimer(); + const mime = recorder.mimeType || mimeType || 'audio/webm'; + const blob = new Blob(audioChunksRef.current, { type: mime }); + audioChunksRef.current = []; + + // 释放麦克风 + if (mediaStreamRef.current) { + mediaStreamRef.current.getTracks().forEach((t) => t.stop()); + mediaStreamRef.current = null; + } + + if (blob.size < 1024) { + enterError(new Error('录音过短，请重试')); + return; + } + + setVoiceState('transcribing'); + try { + const ext = getAudioFileExtension(mime); + const file = new File([blob], `recording.${ext}`, { type: mime }); + const result = await interviewApi.transcribeVoice(file); + const cleaned = cleanTranscript(result.text); + if (cleaned) onCommitRef.current(cleaned); + setVoiceState('idle'); + } catch (err) { + enterError(err); + } + }; + + recorder.onerror = (event: Event) => { + const error = (event as unknown as { error?: { message?: string } }).error; + enterError(new Error(`MediaRecorder error: ${error?.message ?? 'unknown'}`)); + }; + + recorder.start(1000); + setVoiceState('recording'); + startTimer(); + }, [enterError, startTimer, stopTimer]); + + const stopBatch = useCallback(async () => { + const recorder = mediaRecorderRef.current; + if (recorder && recorder.state !== 'inactive') { + recorder.stop(); // onstop 触发 transcribe + } + }, []); + + // ============================ 公开 API ============================ + const start = useCallback(async () => { + try { + if (mode === 'stream') { + await startStream(); + } else { + await startBatch(); + } + } catch (err) { + enterError(err); + } + }, [mode, startStream, startBatch, enterError]); + + const stop = useCallback(async () => { + try { + if (mode === 'stream') { + await stopStream(); + } else { + await stopBatch(); + } + } catch (err) { + enterError(err); + } + }, [mode, stopStream, stopBatch, enterError]); + + const cancel = useCallback(() => { + stopTimer(); + if (errorRecoverTimerRef.current !== null) { + window.clearTimeout(errorRecoverTimerRef.current); + errorRecoverTimerRef.current = null; + } + // 流式 + if (wsClientRef.current) { + wsClientRef.current.close(); + wsClientRef.current = null; + } + if (workletNodeRef.current) { + workletNodeRef.current.disconnect(); + workletNodeRef.current = null; + } + if (sourceNodeRef.current) { + sourceNodeRef.current.disconnect(); + sourceNodeRef.current = null; + } + if (audioContextRef.current) { + audioContextRef.current.close().catch(() => undefined); + audioContextRef.current = null; + } + pcmBufferRef.current = []; + // 批式 + if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') { + try { + mediaRecorderRef.current.ondataavailable = null; + mediaRecorderRef.current.onstop = null; + mediaRecorderRef.current.stop(); + } catch { + // ignore + } + mediaRecorderRef.current = null; + } + audioChunksRef.current = []; + if (mediaStreamRef.current) { + mediaStreamRef.current.getTracks().forEach((t) => t.stop()); + mediaStreamRef.current = null; + } + setVoiceState('idle'); + setVoiceError(''); + setPartialText(''); + setRecordingSeconds(0); + }, [stopTimer]); + + // 卸载清理 + useEffect(() => { + return () => { + cancel(); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + return { + voiceState, + voiceError, + recordingSeconds, + partialText, + start, + stop, + cancel, + }; +} diff --git a/frontend/src/pages/InterviewPage.tsx b/frontend/src/pages/InterviewPage.tsx index 54c50b8..508f6e0 100644 --- a/frontend/src/pages/InterviewPage.tsx +++ b/frontend/src/pages/InterviewPage.tsx @@ -7,8 +7,6 @@ import { AlertCircle, CheckCircle2, Clock3, - Mic, - MicOff, Lightbulb, RotateCcw, Trophy, @@ -16,6 +14,10 @@ import { Library, } from 'lucide-react'; import { interviewApi } from '../api/interview'; +import { VoiceMicButton } from '../components/VoiceMicButton'; +import { VoiceStatusLine } from '../components/VoiceStatusLine'; +import { useVoiceInput } from '../hooks/useVoiceInput'; +import { cleanTranscript } from '../utils/voice'; import type { DynamicCoachHint, DynamicReportDTO, @@ -33,34 +35,6 @@ const PROCESSING_STATUSES = new Set(['PENDING', 'PROCESSING']); type VoiceState = 'idle' | 'recording' | 'transcribing'; type DynamicReviewItem = { turn: DynamicTurnDTO; answer: string; score: number | null }; -const VOICE_ERROR_MESSAGES: Record = { - NotAllowedError: '麦克风权限被拒绝，请在浏览器地址栏允许后再试', - NotFoundError: '没有检测到可用麦克风', - NotReadableError: '麦克风正在被其他应用占用', - SecurityError: '当前页面不允许访问麦克风', -}; - -const getPreferredAudioMimeType = () => { - if (typeof MediaRecorder === 'undefined') return ''; - const candidates = [ - 'audio/webm;codecs=opus', - 'audio/webm', - 'audio/ogg;codecs=opus', - 'audio/mp4', - ]; - return candidates.find(type => MediaRecorder.isTypeSupported(type)) || ''; -}; - -const getAudioFileExtension = (mimeType: string) => { - if (mimeType.includes('ogg')) return 'ogg'; - if (mimeType.includes('mp4')) return 'm4a'; - if (mimeType.includes('mpeg')) return 'mp3'; - if (mimeType.includes('wav')) return 'wav'; - return 'webm'; -}; - -const cleanTranscript = (text: string) => text.replace(/\s+/g, ' ').trim(); - const dynamicTypeLabel: Record = { PROJECT: '项目', KNOWLEDGE: '知识', @@ -160,9 +134,6 @@ export default function InterviewPage() { const [loading, setLoading] = useState(true); const [submitting, setSubmitting] = useState(false); const [error, setError] = useState(''); - const [voiceState, setVoiceState] = useState('idle'); - const [voiceError, setVoiceError] = useState(''); - const [recordingSeconds, setRecordingSeconds] = useState(0); const [completed, setCompleted] = useState(false); const [report, setReport] = useState(null); const [questionHistory, setQuestionHistory] = useState<{ question: InterviewQuestionDTO; answer: string }[]>([]); @@ -171,11 +142,13 @@ export default function InterviewPage() { const [ragLoadingTopicId, setRagLoadingTopicId] = useState(null); const [retryingDynamicTopicId, setRetryingDynamicTopicId] = useState(null); const textareaRef = useRef(null); - const mediaRecorderRef = useRef(null); - const mediaStreamRef = useRef(null); - const audioChunksRef = useRef([]); - const recordingTimerRef = useRef(null); - const shouldTranscribeRef = useRef(false); + + // 流式 STT hook（mode: 'stream'，失败时 UI 层可考虑切回 'batch'） + const voice = useVoiceInput({ + mode: 'stream', + onCommit: (text) => appendTranscript(text), + getToken: () => localStorage.getItem('access_token'), + }); useEffect(() => { if (!sessionId) { @@ -215,12 +188,7 @@ export default function InterviewPage() { useEffect(() => { return () => { - shouldTranscribeRef.current = false; - if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') { - mediaRecorderRef.current.stop(); - } - releaseVoiceStream(); - clearRecordingTimer(); + // voice hook 自己管理麦克风 / AudioContext / WS 清理 }; }, []); @@ -344,124 +312,20 @@ export default function InterviewPage() { }; const clearRecordingTimer = () => { - if (recordingTimerRef.current !== null) { - window.clearInterval(recordingTimerRef.current); - recordingTimerRef.current = null; - } + // 保留占位（旧 hook 残留清理），由 voice hook 自身管理 }; const releaseVoiceStream = () => { - mediaStreamRef.current?.getTracks().forEach(track => track.stop()); - mediaStreamRef.current = null; - }; - - const transcribeAudioBlob = async (blob: Blob) => { - if (blob.size < 1024) { - setVoiceError('录音时间太短，可以再说一次'); - setVoiceState('idle'); - return; - } - - setVoiceState('transcribing'); - setVoiceError(''); - try { - const mimeType = blob.type || 'audio/webm'; - const extension = getAudioFileExtension(mimeType); - const audioFile = new File([blob], `interview-answer-${Date.now()}.${extension}`, { type: mimeType }); - const result = await interviewApi.transcribeVoice(audioFile); - const transcript = cleanTranscript(result.text); - if (!transcript) { - setVoiceError('这段录音没有识别出文字，可以靠近麦克风再试一次'); - return; - } - appendTranscript(transcript); - } catch (err) { - setVoiceError(err instanceof Error ? err.message : '语音转文字失败，请重新录音或手动输入'); - } finally { - setVoiceState('idle'); - setRecordingSeconds(0); - } - }; - - const stopVoiceRecording = () => { - const recorder = mediaRecorderRef.current; - if (!recorder || recorder.state === 'inactive') return; - recorder.stop(); - }; - - const startVoiceRecording = async () => { - if (!navigator.mediaDevices?.getUserMedia || typeof MediaRecorder === 'undefined') { - setVoiceError('当前浏览器不支持录音，请换 Chrome/Edge 或手动输入'); - return; - } - - try { - const stream = await navigator.mediaDevices.getUserMedia({ - audio: { - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - }, - }); - const mimeType = getPreferredAudioMimeType(); - const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); - - audioChunksRef.current = []; - shouldTranscribeRef.current = true; - mediaStreamRef.current = stream; - mediaRecorderRef.current = recorder; - - recorder.ondataavailable = event => { - if (event.data.size > 0) { - audioChunksRef.current.push(event.data); - } - }; - - recorder.onerror = () => { - setVoiceError('录音失败，请重新授权麦克风或手动输入'); - setVoiceState('idle'); - clearRecordingTimer(); - releaseVoiceStream(); - }; - - recorder.onstop = () => { - clearRecordingTimer(); - releaseVoiceStream(); - const shouldTranscribe = shouldTranscribeRef.current; - shouldTranscribeRef.current = false; - mediaRecorderRef.current = null; - - if (!shouldTranscribe) return; - const audioType = recorder.mimeType || mimeType || 'audio/webm'; - const blob = new Blob(audioChunksRef.current, { type: audioType }); - audioChunksRef.current = []; - void transcribeAudioBlob(blob); - }; - - setVoiceError(''); - setRecordingSeconds(0); - setVoiceState('recording'); - recordingTimerRef.current = window.setInterval(() => { - setRecordingSeconds(value => value + 1); - }, 1000); - recorder.start(1000); - } catch (err) { - const name = err instanceof DOMException ? err.name : ''; - setVoiceError(VOICE_ERROR_MESSAGES[name] || '无法打开麦克风，请检查权限后再试'); - setVoiceState('idle'); - clearRecordingTimer(); - releaseVoiceStream(); - } + // 保留占位（旧 hook 残留清理），由 voice hook 自身管理 }; const toggleVoiceInput = () => { - if (voiceState === 'recording') { - stopVoiceRecording(); - return; - } - if (voiceState === 'idle') { - void startVoiceRecording(); + if (voice.voiceState === 'idle' || voice.voiceState === 'error') { + void voice.start(); + } else if (voice.voiceState === 'recording' || voice.voiceState === 'streaming') { + void voice.stop(); } + // transcribing 状态：禁止切换 }; const loadRagInsight = async (topicId: number | null | undefined) => { @@ -513,7 +377,7 @@ export default function InterviewPage() { return; } - if (!sessionId || !answer.trim() || submitting || !currentQuestion || voiceState !== 'idle') return; + if (!sessionId || !answer.trim() || submitting || !currentQuestion || voice.voiceState !== 'idle') return; setSubmitting(true); setError(''); try { @@ -538,7 +402,7 @@ export default function InterviewPage() { }; const handleDynamicSubmit = async () => { - if (!sessionId || !answer.trim() || submitting || !dynamicTurn?.id || voiceState !== 'idle') return; + if (!sessionId || !answer.trim() || submitting || !dynamicTurn?.id || voice.voiceState !== 'idle') return; setSubmitting(true); setError(''); try { @@ -1100,32 +964,25 @@ export default function InterviewPage() {
- + disabled={submitting} + /> Ctrl + Enter 提交 - {voiceState === 'recording' && 录音中 {recordingSeconds}s} +
- {voiceError &&

{voiceError}

}
+ disabled={submitting} + /> Ctrl + Enter 提交 - {voiceState === 'recording' && ( - 录音中 {recordingSeconds}s - )} - {voiceState === 'transcribing' && 正在转文字...} +
- {voiceState === 'idle' && !voiceError && ( -

点击录音，说完后停止，会自动转成文字并填入回答框。

- )} - {voiceError &&

{voiceError}

}