初始化提交
This commit is contained in:
chenxin committed
commit
887c1f449e
963 files changed
+166869
No files matched your search
@@ -0,0 +1,325 @@
|
||||
"""MiMo TTS adapter — ``POST /v1/chat/completions`` with ``audio`` parameter.
|
||||
|
||||
MiMo 的 TTS 不走独立音频端点，而是通过聊天补全接口扩展实现：
|
||||
|
||||
POST {base_url}/chat/completions
|
||||
Content-Type: application/json
|
||||
Authorization: Bearer {api_key}
|
||||
|
||||
{
|
||||
"model": "mimo-v2.5-tts",
|
||||
"messages": [
|
||||
{"role": "user", "content": "（可选）音色风格描述，voicedesign 必填"},
|
||||
{"role": "assistant", "content": "要朗读的文本"}
|
||||
],
|
||||
"audio": {
|
||||
"format": "wav",
|
||||
"voice": "mimo_default"
|
||||
}
|
||||
}
|
||||
|
||||
响应：:
|
||||
|
||||
{
|
||||
"choices": [{
|
||||
"message": {
|
||||
"audio": {
|
||||
"data": "<base64 编码的音频数据>",
|
||||
"id": "...",
|
||||
"expires_at": null,
|
||||
"transcript": null
|
||||
}
|
||||
}
|
||||
}]
|
||||
}
|
||||
|
||||
模型差异：
|
||||
|
||||
- ``mimo-v2.5-tts`` — ``voice`` 可选，默认 ``mimo_default``，仅支持预置音色
|
||||
- ``mimo-v2.5-tts-voiceclone`` — ``voice`` 必填，传入音频样本的 base64（mp3/wav）
|
||||
- ``mimo-v2.5-tts-voicedesign`` — 不支持 ``voice``，\n ``user`` 消息必填（风格描述），支持 ``optimize_text_preview``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
from app.ai_provider.audio.base import (
|
||||
AudioAdapterBase,
|
||||
AudioServiceError,
|
||||
TTSProtocol,
|
||||
TTSRequest,
|
||||
TTSResponse,
|
||||
)
|
||||
from app.core.logging import logger
|
||||
|
||||
# voicedesign 模型的默认风格描述（用户未提供时使用）
|
||||
_DEFAULT_VOICE_DESIGN_PROMPT = "自然、清晰的朗读风格，语气平和，语速适中，适合日常对话场景。"
|
||||
|
||||
# TTS 音频缓存：默认最多 256 条，TTL 1 小时
|
||||
_TTS_CACHE_MAX_SIZE = 256
|
||||
_TTS_CACHE_TTL_SECONDS = 3600
|
||||
|
||||
|
||||
class _TTSCacheEntry:
|
||||
__slots__ = ("data", "format", "stored_at")
|
||||
|
||||
def __init__(self, data: bytes, format: str) -> None:
|
||||
self.data = data
|
||||
self.format = format
|
||||
self.stored_at = time.monotonic()
|
||||
|
||||
|
||||
_tts_cache: dict[str, _TTSCacheEntry] = {}
|
||||
|
||||
|
||||
def _tts_cache_key(model: str, voice: str, fmt: str, text: str, extra_json: str) -> str:
|
||||
"""生成 TTS 缓存键：SHA256(model + voice + format + text + extra)。"""
|
||||
raw = f"{model}\0{voice}\0{fmt}\0{text}\0{extra_json}"
|
||||
return hashlib.sha256(raw.encode()).hexdigest()
|
||||
|
||||
|
||||
def _tts_cache_get(key: str) -> _TTSCacheEntry | None:
|
||||
entry = _tts_cache.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
if time.monotonic() - entry.stored_at > _TTS_CACHE_TTL_SECONDS:
|
||||
del _tts_cache[key]
|
||||
return None
|
||||
return entry
|
||||
|
||||
|
||||
def _tts_cache_set(key: str, entry: _TTSCacheEntry) -> None:
|
||||
if len(_tts_cache) >= _TTS_CACHE_MAX_SIZE:
|
||||
# 淘汰最旧的一半
|
||||
sorted_keys = sorted(_tts_cache.keys(), key=lambda k: _tts_cache[k].stored_at)
|
||||
for old_key in sorted_keys[: _TTS_CACHE_MAX_SIZE // 2]:
|
||||
del _tts_cache[old_key]
|
||||
_tts_cache[key] = entry
|
||||
|
||||
|
||||
class HunjingTTSOpenAI(AudioAdapterBase, TTSProtocol):
|
||||
"""MiMo TTS 适配器，通过 ``POST /chat/completions`` 携带 ``audio`` 参数实现。
|
||||
|
||||
调用兼容 OpenAI 格式的 chat completions 端点，在请求体中添加 ``audio`` 字段，
|
||||
然后从响应中提取并解码 base64 音频数据。
|
||||
"""
|
||||
|
||||
_adapter_type = "tts"
|
||||
_default_timeout = 120
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str = "mimo-v2.5-tts",
|
||||
request_timeout_seconds: int = 120,
|
||||
) -> None:
|
||||
self._base_url = base_url.rstrip("/")
|
||||
self._api_key = api_key
|
||||
self._model = model
|
||||
self._timeout = request_timeout_seconds
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
|
||||
# ── 客户端 ───────────────────────────────────────────────────────────
|
||||
|
||||
@property
|
||||
def _http(self) -> httpx.AsyncClient:
|
||||
if self._client is None or self._client.is_closed:
|
||||
self._client = httpx.AsyncClient(timeout=httpx.Timeout(self._timeout))
|
||||
return self._client
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._client is not None and not self._client.is_closed:
|
||||
await self._client.aclose()
|
||||
|
||||
# ── TTSProtocol ──────────────────────────────────────────────────────
|
||||
|
||||
async def synthesize(self, request: TTSRequest) -> TTSResponse:
|
||||
"""通过 ``POST /chat/completions`` 将文本合成为音频。
|
||||
|
||||
构造 chat 消息和 ``audio`` 参数，匹配 MiMo 兼容 OpenAI 格式的 TTS，
|
||||
然后从响应中解码 base64 音频。
|
||||
|
||||
结果按 ``(model, voice, format, text)`` 在进程内缓存，避免重复 API 调用。
|
||||
"""
|
||||
# ── 缓存查找 ──────────────────────────────────────────────────────
|
||||
cache_key = _tts_cache_key(
|
||||
model=self._model,
|
||||
voice=request.voice,
|
||||
fmt=request.response_format,
|
||||
text=request.text,
|
||||
extra_json=str(sorted((request.extra or {}).items())),
|
||||
)
|
||||
cached = _tts_cache_get(cache_key)
|
||||
if cached is not None:
|
||||
logger.info("audio_tts.cache_hit", model=self._model, text_length=len(request.text))
|
||||
return TTSResponse(audio_data=cached.data, format=cached.format)
|
||||
|
||||
# ── API 调用 ──────────────────────────────────────────────────────
|
||||
url = f"{self._base_url}/chat/completions"
|
||||
|
||||
# 构造 messages
|
||||
messages = self._build_messages(request)
|
||||
|
||||
# 构造 audio 参数
|
||||
audio_params = self._build_audio_params(request)
|
||||
|
||||
payload: dict = {
|
||||
"model": self._model,
|
||||
"messages": messages,
|
||||
"audio": audio_params,
|
||||
"max_completion_tokens": 8192,
|
||||
}
|
||||
# 合并 extra（如 frequency_penalty, temperature 等）
|
||||
if request.extra:
|
||||
payload.update(request.extra)
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
logger.info(
|
||||
"audio_tts.request",
|
||||
model=self._model,
|
||||
voice=request.voice,
|
||||
audio_format=request.response_format,
|
||||
text_length=len(request.text),
|
||||
url=url,
|
||||
)
|
||||
|
||||
try:
|
||||
response = await self._http.post(url, json=payload, headers=headers)
|
||||
except httpx.TimeoutException as exc:
|
||||
raise AudioServiceError(
|
||||
status_code=504,
|
||||
code="audio_tts_timeout",
|
||||
message="TTS request timed out calling chat completions.",
|
||||
) from exc
|
||||
except httpx.RequestError as exc:
|
||||
raise AudioServiceError(
|
||||
status_code=503,
|
||||
code="audio_tts_connection_error",
|
||||
message=f"TTS connection error: {exc}",
|
||||
) from exc
|
||||
|
||||
if response.status_code != 200:
|
||||
body = response.text[:500]
|
||||
raise AudioServiceError(
|
||||
status_code=response.status_code,
|
||||
code="audio_tts_api_error",
|
||||
message=f"TTS API returned {response.status_code}: {body}",
|
||||
)
|
||||
|
||||
# 解析响应，提取 base64 音频
|
||||
audio_data = self._extract_audio(response.json())
|
||||
|
||||
# ── 写入缓存 ──────────────────────────────────────────────────────
|
||||
_tts_cache_set(cache_key, _TTSCacheEntry(audio_data, request.response_format))
|
||||
|
||||
logger.info(
|
||||
"audio_tts.success",
|
||||
model=self._model,
|
||||
audio_bytes=len(audio_data),
|
||||
cache_size=len(_tts_cache),
|
||||
)
|
||||
return TTSResponse(audio_data=audio_data, format=request.response_format)
|
||||
|
||||
# ── 辅助方法 ──────────────────────────────────────────────────────────
|
||||
|
||||
def _build_messages(self, request: TTSRequest) -> list[dict]:
|
||||
"""构造 chat messages 数组。
|
||||
|
||||
- voicedesign 模型：user 消息必填（风格描述）
|
||||
- 其他模型：user 消息可选，assistant 消息存放待朗读文本
|
||||
"""
|
||||
is_voicedesign = "voicedesign" in self._model
|
||||
|
||||
if is_voicedesign:
|
||||
# voicedesign：user 消息为风格提示，assistant 消息为朗读文本
|
||||
style_prompt = (request.extra or {}).pop("style_prompt", _DEFAULT_VOICE_DESIGN_PROMPT)
|
||||
return [
|
||||
{"role": "user", "content": style_prompt},
|
||||
{"role": "assistant", "content": request.text},
|
||||
]
|
||||
# 基础 TTS / voiceclone：只传 assistant 消息
|
||||
return [
|
||||
{"role": "assistant", "content": request.text},
|
||||
]
|
||||
|
||||
def _build_audio_params(self, request: TTSRequest) -> dict:
|
||||
"""构造 ``audio`` 参数字段，按模型处理 voice/format/optimize_text_preview 等。"""
|
||||
params: dict = {
|
||||
"format": _resolve_api_format(request.response_format),
|
||||
}
|
||||
|
||||
is_voicedesign = "voicedesign" in self._model
|
||||
|
||||
if is_voicedesign:
|
||||
# voicedesign 不支持 voice 字段
|
||||
optimize = (request.extra or {}).pop("optimize_text_preview", None)
|
||||
if optimize is not None:
|
||||
params["optimize_text_preview"] = optimize
|
||||
else:
|
||||
# 基础 TTS / voiceclone：按需传 voice
|
||||
if request.voice and request.voice != "default":
|
||||
params["voice"] = request.voice
|
||||
else:
|
||||
params["voice"] = "mimo_default"
|
||||
|
||||
return params
|
||||
|
||||
@staticmethod
|
||||
def _extract_audio(body: dict) -> bytes:
|
||||
"""从 chat completion 响应中提取并解码 base64 音频数据。"""
|
||||
try:
|
||||
choices = body.get("choices", [])
|
||||
if not choices:
|
||||
raise AudioServiceError(
|
||||
status_code=500,
|
||||
code="audio_tts_no_choices",
|
||||
message="TTS response has no choices.",
|
||||
)
|
||||
message = choices[0].get("message", {})
|
||||
audio_info = message.get("audio")
|
||||
if not audio_info:
|
||||
raise AudioServiceError(
|
||||
status_code=500,
|
||||
code="audio_tts_no_audio_field",
|
||||
message="TTS response missing audio field in message.",
|
||||
)
|
||||
raw = audio_info.get("data")
|
||||
if not raw:
|
||||
raise AudioServiceError(
|
||||
status_code=500,
|
||||
code="audio_tts_empty_audio_data",
|
||||
message="TTS response has empty audio.data.",
|
||||
)
|
||||
return base64.b64decode(raw)
|
||||
except AudioServiceError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AudioServiceError(
|
||||
status_code=500,
|
||||
code="audio_tts_response_parse_error",
|
||||
message=f"Failed to parse TTS response: {exc}",
|
||||
) from exc
|
||||
|
||||
|
||||
def _resolve_api_format(user_format: str) -> str:
|
||||
"""将用户层面的格式名映射为 API 期望的格式。
|
||||
|
||||
MiMo TTS API 接受：``wav``、``mp3``、``pcm``、``pcm16``。
|
||||
"""
|
||||
fmt = user_format.lower()
|
||||
if fmt in ("pcm", "pcm16"):
|
||||
return "pcm16"
|
||||
if fmt in ("wav", "mp3"):
|
||||
return fmt
|
||||
return "wav"
|
||||
Reference in new issue
Block a user