"""MiMo TTS adapter — ``POST /v1/chat/completions`` with ``audio`` parameter. MiMo 的 TTS 不走独立音频端点,而是通过聊天补全接口扩展实现: POST {base_url}/chat/completions Content-Type: application/json Authorization: Bearer {api_key} { "model": "mimo-v2.5-tts", "messages": [ {"role": "user", "content": "(可选)音色风格描述,voicedesign 必填"}, {"role": "assistant", "content": "要朗读的文本"} ], "audio": { "format": "wav", "voice": "mimo_default" } } 响应:: { "choices": [{ "message": { "audio": { "data": "", "id": "...", "expires_at": null, "transcript": null } } }] } 模型差异: - ``mimo-v2.5-tts`` — ``voice`` 可选,默认 ``mimo_default``,仅支持预置音色 - ``mimo-v2.5-tts-voiceclone`` — ``voice`` 必填,传入音频样本的 base64(mp3/wav) - ``mimo-v2.5-tts-voicedesign`` — 不支持 ``voice``,\n ``user`` 消息必填(风格描述),支持 ``optimize_text_preview`` """ from __future__ import annotations import base64 import hashlib import time import httpx from app.ai_provider.audio.base import ( AudioAdapterBase, AudioServiceError, TTSProtocol, TTSRequest, TTSResponse, ) from app.core.logging import logger # voicedesign 模型的默认风格描述(用户未提供时使用) _DEFAULT_VOICE_DESIGN_PROMPT = "自然、清晰的朗读风格,语气平和,语速适中,适合日常对话场景。" # TTS 音频缓存:默认最多 256 条,TTL 1 小时 _TTS_CACHE_MAX_SIZE = 256 _TTS_CACHE_TTL_SECONDS = 3600 class _TTSCacheEntry: __slots__ = ("data", "format", "stored_at") def __init__(self, data: bytes, format: str) -> None: self.data = data self.format = format self.stored_at = time.monotonic() _tts_cache: dict[str, _TTSCacheEntry] = {} def _tts_cache_key(model: str, voice: str, fmt: str, text: str, extra_json: str) -> str: """生成 TTS 缓存键:SHA256(model + voice + format + text + extra)。""" raw = f"{model}\0{voice}\0{fmt}\0{text}\0{extra_json}" return hashlib.sha256(raw.encode()).hexdigest() def _tts_cache_get(key: str) -> _TTSCacheEntry | None: entry = _tts_cache.get(key) if entry is None: return None if time.monotonic() - entry.stored_at > _TTS_CACHE_TTL_SECONDS: del _tts_cache[key] return None return entry def _tts_cache_set(key: str, entry: _TTSCacheEntry) -> None: if len(_tts_cache) >= _TTS_CACHE_MAX_SIZE: # 淘汰最旧的一半 sorted_keys = sorted(_tts_cache.keys(), key=lambda k: _tts_cache[k].stored_at) for old_key in sorted_keys[: _TTS_CACHE_MAX_SIZE // 2]: del _tts_cache[old_key] _tts_cache[key] = entry class HunjingTTSOpenAI(AudioAdapterBase, TTSProtocol): """MiMo TTS 适配器,通过 ``POST /chat/completions`` 携带 ``audio`` 参数实现。 调用兼容 OpenAI 格式的 chat completions 端点,在请求体中添加 ``audio`` 字段, 然后从响应中提取并解码 base64 音频数据。 """ _adapter_type = "tts" _default_timeout = 120 def __init__( self, *, base_url: str, api_key: str, model: str = "mimo-v2.5-tts", request_timeout_seconds: int = 120, ) -> None: self._base_url = base_url.rstrip("/") self._api_key = api_key self._model = model self._timeout = request_timeout_seconds self._client: httpx.AsyncClient | None = None # ── 客户端 ─────────────────────────────────────────────────────────── @property def _http(self) -> httpx.AsyncClient: if self._client is None or self._client.is_closed: self._client = httpx.AsyncClient(timeout=httpx.Timeout(self._timeout)) return self._client async def close(self) -> None: if self._client is not None and not self._client.is_closed: await self._client.aclose() # ── TTSProtocol ────────────────────────────────────────────────────── async def synthesize(self, request: TTSRequest) -> TTSResponse: """通过 ``POST /chat/completions`` 将文本合成为音频。 构造 chat 消息和 ``audio`` 参数,匹配 MiMo 兼容 OpenAI 格式的 TTS, 然后从响应中解码 base64 音频。 结果按 ``(model, voice, format, text)`` 在进程内缓存,避免重复 API 调用。 """ # ── 缓存查找 ────────────────────────────────────────────────────── cache_key = _tts_cache_key( model=self._model, voice=request.voice, fmt=request.response_format, text=request.text, extra_json=str(sorted((request.extra or {}).items())), ) cached = _tts_cache_get(cache_key) if cached is not None: logger.info("audio_tts.cache_hit", model=self._model, text_length=len(request.text)) return TTSResponse(audio_data=cached.data, format=cached.format) # ── API 调用 ────────────────────────────────────────────────────── url = f"{self._base_url}/chat/completions" # 构造 messages messages = self._build_messages(request) # 构造 audio 参数 audio_params = self._build_audio_params(request) payload: dict = { "model": self._model, "messages": messages, "audio": audio_params, "max_completion_tokens": 8192, } # 合并 extra(如 frequency_penalty, temperature 等) if request.extra: payload.update(request.extra) headers = { "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json", } logger.info( "audio_tts.request", model=self._model, voice=request.voice, audio_format=request.response_format, text_length=len(request.text), url=url, ) try: response = await self._http.post(url, json=payload, headers=headers) except httpx.TimeoutException as exc: raise AudioServiceError( status_code=504, code="audio_tts_timeout", message="TTS request timed out calling chat completions.", ) from exc except httpx.RequestError as exc: raise AudioServiceError( status_code=503, code="audio_tts_connection_error", message=f"TTS connection error: {exc}", ) from exc if response.status_code != 200: body = response.text[:500] raise AudioServiceError( status_code=response.status_code, code="audio_tts_api_error", message=f"TTS API returned {response.status_code}: {body}", ) # 解析响应,提取 base64 音频 audio_data = self._extract_audio(response.json()) # ── 写入缓存 ────────────────────────────────────────────────────── _tts_cache_set(cache_key, _TTSCacheEntry(audio_data, request.response_format)) logger.info( "audio_tts.success", model=self._model, audio_bytes=len(audio_data), cache_size=len(_tts_cache), ) return TTSResponse(audio_data=audio_data, format=request.response_format) # ── 辅助方法 ────────────────────────────────────────────────────────── def _build_messages(self, request: TTSRequest) -> list[dict]: """构造 chat messages 数组。 - voicedesign 模型:user 消息必填(风格描述) - 其他模型:user 消息可选,assistant 消息存放待朗读文本 """ is_voicedesign = "voicedesign" in self._model if is_voicedesign: # voicedesign:user 消息为风格提示,assistant 消息为朗读文本 style_prompt = (request.extra or {}).pop("style_prompt", _DEFAULT_VOICE_DESIGN_PROMPT) return [ {"role": "user", "content": style_prompt}, {"role": "assistant", "content": request.text}, ] # 基础 TTS / voiceclone:只传 assistant 消息 return [ {"role": "assistant", "content": request.text}, ] def _build_audio_params(self, request: TTSRequest) -> dict: """构造 ``audio`` 参数字段,按模型处理 voice/format/optimize_text_preview 等。""" params: dict = { "format": _resolve_api_format(request.response_format), } is_voicedesign = "voicedesign" in self._model if is_voicedesign: # voicedesign 不支持 voice 字段 optimize = (request.extra or {}).pop("optimize_text_preview", None) if optimize is not None: params["optimize_text_preview"] = optimize else: # 基础 TTS / voiceclone:按需传 voice if request.voice and request.voice != "default": params["voice"] = request.voice else: params["voice"] = "mimo_default" return params @staticmethod def _extract_audio(body: dict) -> bytes: """从 chat completion 响应中提取并解码 base64 音频数据。""" try: choices = body.get("choices", []) if not choices: raise AudioServiceError( status_code=500, code="audio_tts_no_choices", message="TTS response has no choices.", ) message = choices[0].get("message", {}) audio_info = message.get("audio") if not audio_info: raise AudioServiceError( status_code=500, code="audio_tts_no_audio_field", message="TTS response missing audio field in message.", ) raw = audio_info.get("data") if not raw: raise AudioServiceError( status_code=500, code="audio_tts_empty_audio_data", message="TTS response has empty audio.data.", ) return base64.b64decode(raw) except AudioServiceError: raise except Exception as exc: raise AudioServiceError( status_code=500, code="audio_tts_response_parse_error", message=f"Failed to parse TTS response: {exc}", ) from exc def _resolve_api_format(user_format: str) -> str: """将用户层面的格式名映射为 API 期望的格式。 MiMo TTS API 接受:``wav``、``mp3``、``pcm``、``pcm16``。 """ fmt = user_format.lower() if fmt in ("pcm", "pcm16"): return "pcm16" if fmt in ("wav", "mp3"): return fmt return "wav"