326 lines
12 KiB
Python
326 lines
12 KiB
Python
"""MiMo TTS adapter — ``POST /v1/chat/completions`` with ``audio`` parameter.
|
||
|
||
MiMo 的 TTS 不走独立音频端点，而是通过聊天补全接口扩展实现：
|
||
|
||
POST {base_url}/chat/completions
|
||
Content-Type: application/json
|
||
Authorization: Bearer {api_key}
|
||
|
||
{
|
||
"model": "mimo-v2.5-tts",
|
||
"messages": [
|
||
{"role": "user", "content": "（可选）音色风格描述，voicedesign 必填"},
|
||
{"role": "assistant", "content": "要朗读的文本"}
|
||
],
|
||
"audio": {
|
||
"format": "wav",
|
||
"voice": "mimo_default"
|
||
}
|
||
}
|
||
|
||
响应：:
|
||
|
||
{
|
||
"choices": [{
|
||
"message": {
|
||
"audio": {
|
||
"data": "<base64 编码的音频数据>",
|
||
"id": "...",
|
||
"expires_at": null,
|
||
"transcript": null
|
||
}
|
||
}
|
||
}]
|
||
}
|
||
|
||
模型差异：
|
||
|
||
- ``mimo-v2.5-tts`` — ``voice`` 可选，默认 ``mimo_default``，仅支持预置音色
|
||
- ``mimo-v2.5-tts-voiceclone`` — ``voice`` 必填，传入音频样本的 base64（mp3/wav）
|
||
- ``mimo-v2.5-tts-voicedesign`` — 不支持 ``voice``，\n ``user`` 消息必填（风格描述），支持 ``optimize_text_preview``
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import base64
|
||
import hashlib
|
||
import time
|
||
|
||
import httpx
|
||
|
||
from app.ai_provider.audio.base import (
|
||
AudioAdapterBase,
|
||
AudioServiceError,
|
||
TTSProtocol,
|
||
TTSRequest,
|
||
TTSResponse,
|
||
)
|
||
from app.core.logging import logger
|
||
|
||
# voicedesign 模型的默认风格描述（用户未提供时使用）
|
||
_DEFAULT_VOICE_DESIGN_PROMPT = "自然、清晰的朗读风格，语气平和，语速适中，适合日常对话场景。"
|
||
|
||
# TTS 音频缓存：默认最多 256 条，TTL 1 小时
|
||
_TTS_CACHE_MAX_SIZE = 256
|
||
_TTS_CACHE_TTL_SECONDS = 3600
|
||
|
||
|
||
class _TTSCacheEntry:
|
||
__slots__ = ("data", "format", "stored_at")
|
||
|
||
def __init__(self, data: bytes, format: str) -> None:
|
||
self.data = data
|
||
self.format = format
|
||
self.stored_at = time.monotonic()
|
||
|
||
|
||
_tts_cache: dict[str, _TTSCacheEntry] = {}
|
||
|
||
|
||
def _tts_cache_key(model: str, voice: str, fmt: str, text: str, extra_json: str) -> str:
|
||
"""生成 TTS 缓存键：SHA256(model + voice + format + text + extra)。"""
|
||
raw = f"{model}\0{voice}\0{fmt}\0{text}\0{extra_json}"
|
||
return hashlib.sha256(raw.encode()).hexdigest()
|
||
|
||
|
||
def _tts_cache_get(key: str) -> _TTSCacheEntry | None:
|
||
entry = _tts_cache.get(key)
|
||
if entry is None:
|
||
return None
|
||
if time.monotonic() - entry.stored_at > _TTS_CACHE_TTL_SECONDS:
|
||
del _tts_cache[key]
|
||
return None
|
||
return entry
|
||
|
||
|
||
def _tts_cache_set(key: str, entry: _TTSCacheEntry) -> None:
|
||
if len(_tts_cache) >= _TTS_CACHE_MAX_SIZE:
|
||
# 淘汰最旧的一半
|
||
sorted_keys = sorted(_tts_cache.keys(), key=lambda k: _tts_cache[k].stored_at)
|
||
for old_key in sorted_keys[: _TTS_CACHE_MAX_SIZE // 2]:
|
||
del _tts_cache[old_key]
|
||
_tts_cache[key] = entry
|
||
|
||
|
||
class HunjingTTSOpenAI(AudioAdapterBase, TTSProtocol):
|
||
"""MiMo TTS 适配器，通过 ``POST /chat/completions`` 携带 ``audio`` 参数实现。
|
||
|
||
调用兼容 OpenAI 格式的 chat completions 端点，在请求体中添加 ``audio`` 字段，
|
||
然后从响应中提取并解码 base64 音频数据。
|
||
"""
|
||
|
||
_adapter_type = "tts"
|
||
_default_timeout = 120
|
||
|
||
def __init__(
|
||
self,
|
||
*,
|
||
base_url: str,
|
||
api_key: str,
|
||
model: str = "mimo-v2.5-tts",
|
||
request_timeout_seconds: int = 120,
|
||
) -> None:
|
||
self._base_url = base_url.rstrip("/")
|
||
self._api_key = api_key
|
||
self._model = model
|
||
self._timeout = request_timeout_seconds
|
||
self._client: httpx.AsyncClient | None = None
|
||
|
||
# ── 客户端 ───────────────────────────────────────────────────────────
|
||
|
||
@property
|
||
def _http(self) -> httpx.AsyncClient:
|
||
if self._client is None or self._client.is_closed:
|
||
self._client = httpx.AsyncClient(timeout=httpx.Timeout(self._timeout))
|
||
return self._client
|
||
|
||
async def close(self) -> None:
|
||
if self._client is not None and not self._client.is_closed:
|
||
await self._client.aclose()
|
||
|
||
# ── TTSProtocol ──────────────────────────────────────────────────────
|
||
|
||
async def synthesize(self, request: TTSRequest) -> TTSResponse:
|
||
"""通过 ``POST /chat/completions`` 将文本合成为音频。
|
||
|
||
构造 chat 消息和 ``audio`` 参数，匹配 MiMo 兼容 OpenAI 格式的 TTS，
|
||
然后从响应中解码 base64 音频。
|
||
|
||
结果按 ``(model, voice, format, text)`` 在进程内缓存，避免重复 API 调用。
|
||
"""
|
||
# ── 缓存查找 ──────────────────────────────────────────────────────
|
||
cache_key = _tts_cache_key(
|
||
model=self._model,
|
||
voice=request.voice,
|
||
fmt=request.response_format,
|
||
text=request.text,
|
||
extra_json=str(sorted((request.extra or {}).items())),
|
||
)
|
||
cached = _tts_cache_get(cache_key)
|
||
if cached is not None:
|
||
logger.info("audio_tts.cache_hit", model=self._model, text_length=len(request.text))
|
||
return TTSResponse(audio_data=cached.data, format=cached.format)
|
||
|
||
# ── API 调用 ──────────────────────────────────────────────────────
|
||
url = f"{self._base_url}/chat/completions"
|
||
|
||
# 构造 messages
|
||
messages = self._build_messages(request)
|
||
|
||
# 构造 audio 参数
|
||
audio_params = self._build_audio_params(request)
|
||
|
||
payload: dict = {
|
||
"model": self._model,
|
||
"messages": messages,
|
||
"audio": audio_params,
|
||
"max_completion_tokens": 8192,
|
||
}
|
||
# 合并 extra（如 frequency_penalty, temperature 等）
|
||
if request.extra:
|
||
payload.update(request.extra)
|
||
|
||
headers = {
|
||
"Authorization": f"Bearer {self._api_key}",
|
||
"Content-Type": "application/json",
|
||
}
|
||
|
||
logger.info(
|
||
"audio_tts.request",
|
||
model=self._model,
|
||
voice=request.voice,
|
||
audio_format=request.response_format,
|
||
text_length=len(request.text),
|
||
url=url,
|
||
)
|
||
|
||
try:
|
||
response = await self._http.post(url, json=payload, headers=headers)
|
||
except httpx.TimeoutException as exc:
|
||
raise AudioServiceError(
|
||
status_code=504,
|
||
code="audio_tts_timeout",
|
||
message="TTS request timed out calling chat completions.",
|
||
) from exc
|
||
except httpx.RequestError as exc:
|
||
raise AudioServiceError(
|
||
status_code=503,
|
||
code="audio_tts_connection_error",
|
||
message=f"TTS connection error: {exc}",
|
||
) from exc
|
||
|
||
if response.status_code != 200:
|
||
body = response.text[:500]
|
||
raise AudioServiceError(
|
||
status_code=response.status_code,
|
||
code="audio_tts_api_error",
|
||
message=f"TTS API returned {response.status_code}: {body}",
|
||
)
|
||
|
||
# 解析响应，提取 base64 音频
|
||
audio_data = self._extract_audio(response.json())
|
||
|
||
# ── 写入缓存 ──────────────────────────────────────────────────────
|
||
_tts_cache_set(cache_key, _TTSCacheEntry(audio_data, request.response_format))
|
||
|
||
logger.info(
|
||
"audio_tts.success",
|
||
model=self._model,
|
||
audio_bytes=len(audio_data),
|
||
cache_size=len(_tts_cache),
|
||
)
|
||
return TTSResponse(audio_data=audio_data, format=request.response_format)
|
||
|
||
# ── 辅助方法 ──────────────────────────────────────────────────────────
|
||
|
||
def _build_messages(self, request: TTSRequest) -> list[dict]:
|
||
"""构造 chat messages 数组。
|
||
|
||
- voicedesign 模型：user 消息必填（风格描述）
|
||
- 其他模型：user 消息可选，assistant 消息存放待朗读文本
|
||
"""
|
||
is_voicedesign = "voicedesign" in self._model
|
||
|
||
if is_voicedesign:
|
||
# voicedesign：user 消息为风格提示，assistant 消息为朗读文本
|
||
style_prompt = (request.extra or {}).pop("style_prompt", _DEFAULT_VOICE_DESIGN_PROMPT)
|
||
return [
|
||
{"role": "user", "content": style_prompt},
|
||
{"role": "assistant", "content": request.text},
|
||
]
|
||
# 基础 TTS / voiceclone：只传 assistant 消息
|
||
return [
|
||
{"role": "assistant", "content": request.text},
|
||
]
|
||
|
||
def _build_audio_params(self, request: TTSRequest) -> dict:
|
||
"""构造 ``audio`` 参数字段，按模型处理 voice/format/optimize_text_preview 等。"""
|
||
params: dict = {
|
||
"format": _resolve_api_format(request.response_format),
|
||
}
|
||
|
||
is_voicedesign = "voicedesign" in self._model
|
||
|
||
if is_voicedesign:
|
||
# voicedesign 不支持 voice 字段
|
||
optimize = (request.extra or {}).pop("optimize_text_preview", None)
|
||
if optimize is not None:
|
||
params["optimize_text_preview"] = optimize
|
||
else:
|
||
# 基础 TTS / voiceclone：按需传 voice
|
||
if request.voice and request.voice != "default":
|
||
params["voice"] = request.voice
|
||
else:
|
||
params["voice"] = "mimo_default"
|
||
|
||
return params
|
||
|
||
@staticmethod
|
||
def _extract_audio(body: dict) -> bytes:
|
||
"""从 chat completion 响应中提取并解码 base64 音频数据。"""
|
||
try:
|
||
choices = body.get("choices", [])
|
||
if not choices:
|
||
raise AudioServiceError(
|
||
status_code=500,
|
||
code="audio_tts_no_choices",
|
||
message="TTS response has no choices.",
|
||
)
|
||
message = choices[0].get("message", {})
|
||
audio_info = message.get("audio")
|
||
if not audio_info:
|
||
raise AudioServiceError(
|
||
status_code=500,
|
||
code="audio_tts_no_audio_field",
|
||
message="TTS response missing audio field in message.",
|
||
)
|
||
raw = audio_info.get("data")
|
||
if not raw:
|
||
raise AudioServiceError(
|
||
status_code=500,
|
||
code="audio_tts_empty_audio_data",
|
||
message="TTS response has empty audio.data.",
|
||
)
|
||
return base64.b64decode(raw)
|
||
except AudioServiceError:
|
||
raise
|
||
except Exception as exc:
|
||
raise AudioServiceError(
|
||
status_code=500,
|
||
code="audio_tts_response_parse_error",
|
||
message=f"Failed to parse TTS response: {exc}",
|
||
) from exc
|
||
|
||
|
||
def _resolve_api_format(user_format: str) -> str:
|
||
"""将用户层面的格式名映射为 API 期望的格式。
|
||
|
||
MiMo TTS API 接受：``wav``、``mp3``、``pcm``、``pcm16``。
|
||
"""
|
||
fmt = user_format.lower()
|
||
if fmt in ("pcm", "pcm16"):
|
||
return "pcm16"
|
||
if fmt in ("wav", "mp3"):
|
||
return fmt
|
||
return "wav"
|