276 lines
10 KiB
Python
276 lines
10 KiB
Python
"""音频相关 API 端点 — TTS（文本转语音）、ASR（语音转文本）等。
|
||
|
||
音频端点使用神识的模型注册表查找可用的音频模型，并调用对应音频适配器合成 /
|
||
转写。ASR 与 TTS 按 ``tags`` 中的 ``asr`` / ``tts`` 能力区分，ASR 内部再按
|
||
``provider`` 区分协议：``yuanqi`` 走 multipart ``/v1/audio/transcriptions``，
|
||
其他走 MiMo 风格的 ``/chat/completions`` + ``input_audio``。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from fastapi import APIRouter, File, Form, HTTPException, Response, UploadFile
|
||
from pydantic import BaseModel, Field
|
||
|
||
from app.ai_provider.audio import AudioService, AudioServiceError
|
||
from app.ai_provider.manager import ProviderManagerFactory
|
||
from app.core.logging import logger
|
||
|
||
router = APIRouter(tags=["audio"])
|
||
|
||
_MIME_MAP = {
|
||
"wav": "audio/wav",
|
||
"mp3": "audio/mpeg",
|
||
"opus": "audio/opus",
|
||
"flac": "audio/flac",
|
||
}
|
||
|
||
|
||
class _AudioModelFactory:
|
||
"""Adapter: wraps ProviderManager to provide ModelFactory interface for AudioService.
|
||
|
||
``create_audio(name)`` 按 ``tags`` 判断 ``asr`` / ``tts`` 能力：
|
||
- ``tts`` → ``HunjingTTSOpenAI``（``/chat/completions`` + ``audio``）
|
||
- ``asr`` → 按 ``provider`` 分派：
|
||
- ``yuanqi`` → ``YuanqiASRWhisper``（multipart ``/v1/audio/transcriptions``）
|
||
- 其他 → ``HunjingASROpenAI``（``/chat/completions`` + ``input_audio``）
|
||
"""
|
||
|
||
def __init__(self, manager) -> None:
|
||
self._manager = manager
|
||
|
||
@property
|
||
def registry(self):
|
||
return self._manager.model_registry
|
||
|
||
def create_audio(self, name: str):
|
||
model = self._manager.get_model_config(name)
|
||
is_asr = "asr" in model.tags
|
||
|
||
if is_asr:
|
||
return self._create_asr(model)
|
||
return self._create_tts(model)
|
||
|
||
def _provider_or_none(self, model):
|
||
"""安全获取 provider 配置（provider 未配置时返回 None，走纯端点解析）。"""
|
||
try:
|
||
return self._manager.get_provider_config(model.provider)
|
||
except Exception:
|
||
return None
|
||
|
||
# ── ASR ─────────────────────────────────────────────────────────────
|
||
|
||
def _create_asr(self, model):
|
||
"""按 provider 选择 ASR 协议适配器。"""
|
||
# yuanqi 本地引擎走 multipart /v1/audio/transcriptions
|
||
if model.provider == "yuanqi":
|
||
from app.ai_provider.audio.yuanqi_asr import YuanqiASRWhisper
|
||
|
||
if model.endpoints:
|
||
return YuanqiASRWhisper.from_model_config(
|
||
model, provider=self._provider_or_none(model)
|
||
)
|
||
|
||
provider = self._manager.get_provider_config(model.provider)
|
||
api_key, base_url = self._resolve_credentials(provider)
|
||
if not base_url:
|
||
raise AudioServiceError(
|
||
status_code=500,
|
||
code="audio_asr_no_base_url",
|
||
message=f"Provider '{model.provider}' has no base_url configured.",
|
||
)
|
||
return YuanqiASRWhisper(
|
||
base_url=base_url,
|
||
api_key=api_key or "",
|
||
model=model.model,
|
||
request_timeout_seconds=model.timeout_seconds or 300,
|
||
)
|
||
|
||
# MiMo 等走 /chat/completions + input_audio
|
||
from app.ai_provider.audio.asr import HunjingASROpenAI
|
||
|
||
if model.endpoints:
|
||
return HunjingASROpenAI.from_model_config(
|
||
model, provider=self._provider_or_none(model)
|
||
)
|
||
|
||
provider = self._manager.get_provider_config(model.provider)
|
||
api_key, base_url = self._resolve_credentials(provider)
|
||
if not base_url:
|
||
raise AudioServiceError(
|
||
status_code=500,
|
||
code="audio_asr_no_base_url",
|
||
message=f"Provider '{model.provider}' has no base_url configured.",
|
||
)
|
||
return HunjingASROpenAI(
|
||
base_url=base_url,
|
||
api_key=api_key or "",
|
||
model=model.model,
|
||
request_timeout_seconds=model.timeout_seconds or 300,
|
||
)
|
||
|
||
# ── TTS ─────────────────────────────────────────────────────────────
|
||
|
||
def _create_tts(self, model):
|
||
from app.ai_provider.audio.tts import HunjingTTSOpenAI
|
||
|
||
# 有 endpoints 时交给 from_model_config（按端点的 status/model/超时）
|
||
if model.endpoints:
|
||
return HunjingTTSOpenAI.from_model_config(
|
||
model, provider=self._provider_or_none(model)
|
||
)
|
||
|
||
# 无 endpoint 时回退到 provider 级别凭证
|
||
provider = self._manager.get_provider_config(model.provider)
|
||
api_key, base_url = self._resolve_credentials(provider)
|
||
if not base_url:
|
||
raise AudioServiceError(
|
||
status_code=500,
|
||
code="audio_tts_no_base_url",
|
||
message=f"Provider '{model.provider}' has no base_url configured.",
|
||
)
|
||
return HunjingTTSOpenAI(
|
||
base_url=base_url,
|
||
api_key=api_key or "",
|
||
model=model.model,
|
||
request_timeout_seconds=model.timeout_seconds or 120,
|
||
)
|
||
|
||
def _resolve_credentials(self, provider):
|
||
"""从 settings 解析 provider 级别的 API key 和 base URL。"""
|
||
from app.core.config import settings
|
||
|
||
api_key = (
|
||
settings.__getattribute__(provider.api_key_env.lower())
|
||
if hasattr(settings, provider.api_key_env.lower())
|
||
else ""
|
||
)
|
||
base_url = (
|
||
settings.__getattribute__(provider.base_url_env.lower())
|
||
if hasattr(settings, provider.base_url_env.lower())
|
||
else None
|
||
)
|
||
return api_key, base_url
|
||
|
||
|
||
class TTSRequest(BaseModel):
|
||
"""TTS 合成请求。"""
|
||
|
||
text: str = Field(..., min_length=1, max_length=4096, description="要合成语音的文本。")
|
||
voice: str = Field("default", description="语音角色标识（取决于模型，如 default 或声纹 ID）。")
|
||
format: str = Field("wav", description="音频格式：wav / mp3 / opus / flac。")
|
||
|
||
|
||
@router.post("/audio/tts")
|
||
async def synthesize_speech(body: TTSRequest) -> Response:
|
||
"""合成文本为语音，返回音频二进制流。
|
||
|
||
使用神识注册表中第一个活跃的 TTS 模型（capability: tts）。
|
||
未配置 TTS 模型时返回 503。
|
||
"""
|
||
if not body.text.strip():
|
||
raise HTTPException(status_code=400, detail="text 不能为空。")
|
||
|
||
mime = _MIME_MAP.get(body.format, "audio/wav")
|
||
|
||
try:
|
||
manager = ProviderManagerFactory.create()
|
||
service = AudioService(_AudioModelFactory(manager))
|
||
async with service:
|
||
result = await service.synthesize(
|
||
text=body.text,
|
||
voice=body.voice,
|
||
response_format=body.format,
|
||
)
|
||
except AudioServiceError as exc:
|
||
logger.error(
|
||
"audio.tts.api_error",
|
||
voice=body.voice,
|
||
format=body.format,
|
||
error_code=exc.code,
|
||
error_message=exc.message,
|
||
)
|
||
raise HTTPException(
|
||
status_code=exc.status_code or 502,
|
||
detail=exc.message,
|
||
) from exc
|
||
|
||
logger.info(
|
||
"audio.tts.success",
|
||
voice=body.voice,
|
||
format=body.format,
|
||
audio_bytes=len(result.audio_data),
|
||
)
|
||
|
||
return Response(
|
||
content=result.audio_data,
|
||
media_type=mime,
|
||
headers={
|
||
"Content-Disposition": f'inline; filename="speech.{body.format}"',
|
||
"X-Audio-Format": body.format,
|
||
"Cache-Control": "public, max-age=3600",
|
||
},
|
||
)
|
||
|
||
|
||
# ── ASR（语音转文本）───────────────────────────────────────────────────────
|
||
|
||
|
||
class TranscriptionResponse(BaseModel):
|
||
"""OpenAI Whisper 兼容的转写响应。"""
|
||
|
||
text: str = Field(description="转写后的文本")
|
||
|
||
|
||
@router.post("/audio/transcriptions", response_model=TranscriptionResponse)
|
||
async def transcribe_audio(
|
||
file: UploadFile = File(..., description="音频文件（wav/mp3/m4a/flac/ogg/opus/webm 等）"),
|
||
language: str | None = Form(default=None, description="语言提示（ISO-639-1：zh/en/ja/ko 等），省略则自动检测"),
|
||
model: str | None = Form(default=None, description="可选，指定使用的 ASR 模型逻辑名（如 funasr-sensevoice / mimo-v2.5-asr），省略则用注册表中第一个"),
|
||
) -> TranscriptionResponse:
|
||
"""转写音频为文本，返回 OpenAI Whisper 兼容的 JSON。
|
||
|
||
不指定 ``model`` 时用神识注册表中第一个活跃的 ASR 模型（capability: asr），
|
||
按其 ``provider`` 自动选择协议适配器：``yuanqi`` 走本地 multipart 端点，
|
||
其他走 MiMo 风格的 chat completions。未配置 ASR 模型时返回 503。
|
||
"""
|
||
audio_bytes = await file.read()
|
||
if not audio_bytes:
|
||
raise HTTPException(status_code=400, detail="音频文件为空。")
|
||
|
||
fmt = (file.filename or "audio.wav").rsplit(".", 1)[-1].lower() if "." in (file.filename or "") else "wav"
|
||
|
||
try:
|
||
manager = ProviderManagerFactory.create()
|
||
service = AudioService(_AudioModelFactory(manager))
|
||
async with service:
|
||
result = await service.transcribe(
|
||
audio_data=audio_bytes,
|
||
format=fmt,
|
||
language=language,
|
||
model_name=model,
|
||
)
|
||
except AudioServiceError as exc:
|
||
logger.error(
|
||
"audio.asr.api_error",
|
||
filename=file.filename,
|
||
format=fmt,
|
||
language=language,
|
||
error_code=exc.code,
|
||
error_message=exc.message,
|
||
)
|
||
raise HTTPException(
|
||
status_code=exc.status_code or 502,
|
||
detail=exc.message,
|
||
) from exc
|
||
|
||
logger.info(
|
||
"audio.asr.success",
|
||
filename=file.filename,
|
||
format=fmt,
|
||
language=language,
|
||
text_length=len(result.text),
|
||
text=result.text,
|
||
)
|
||
|
||
return TranscriptionResponse(text=result.text)
|