feat: add video OCR/ASR, DOCX/PDF export, input block and risk config updates
- Video pipeline: video file OCR via VLM plus audio track ASR, integrated into job_handlers with progress emit per phase. New media_utils.py for audio extraction from video files. - Document export: richExport.js replaces inline docx builder; DOCX and PDF export buttons are now enabled in MilkdownEditor. File size limit raised to 100 MB. - Input block: new InputBlockCrepe.vue component with inputBlockPlugin.ts and inputBlock.js for custom user-input nodes in the editor. - Risk config: added Vite dev server ports (5173) to CORS allowlist and increased OCR max input from 10 MB to 100 MB. - TTS/ASR refactor: simplified tts_asr.py model loading and warmup logic. - Test coverage: updated tests for llm, main endpoints, pro completions and web search modules. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
“ydy0615”
+140
-308
@@ -1,260 +1,133 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import wave
|
||||
from typing import Optional
|
||||
|
||||
# 设置 Hugging Face / ModelScope 镜像源为国内镜像
|
||||
os.environ.setdefault("HF_ENDPOINT", "https://hf-mirror.com")
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
asyncio.set_event_loop(asyncio.new_event_loop())
|
||||
import numpy as np # type: ignore
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.debug("numpy import failed: %s", exc)
|
||||
np = None # type: ignore
|
||||
|
||||
try:
|
||||
import torch # type: ignore
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.debug("torch import failed: %s", exc)
|
||||
torch = None # type: ignore
|
||||
|
||||
# New TTS model import
|
||||
try:
|
||||
from qwen_tts import Qwen3TTSModel # type: ignore
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.debug("qwen_tts import failed (optional): %s", e)
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.debug("qwen_tts import failed: %s", exc)
|
||||
Qwen3TTSModel = None # type: ignore
|
||||
|
||||
# ASR model import (MLX-based, Apple Silicon only)
|
||||
try:
|
||||
from mlx_audio.stt.models.qwen3_asr import ( # type: ignore
|
||||
ForcedAlignerModel,
|
||||
Qwen3ASRModel,
|
||||
)
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.debug("mlx_audio import failed (optional): %s", e)
|
||||
Qwen3ASRModel = None # type: ignore
|
||||
ForcedAlignerModel = None # type: ignore
|
||||
from faster_whisper import WhisperModel # type: ignore
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.debug("faster_whisper import failed: %s", exc)
|
||||
WhisperModel = None # type: ignore
|
||||
|
||||
try:
|
||||
from modelscope import snapshot_download # type: ignore
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.debug("modelscope import failed (optional): %s", e)
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.debug("modelscope import failed: %s", exc)
|
||||
snapshot_download = None # type: ignore
|
||||
|
||||
meta_router = APIRouter()
|
||||
generation_router = APIRouter()
|
||||
|
||||
# Global model instances
|
||||
_tts_model: Optional["Qwen3TTSModel"] = None
|
||||
_asr_model: Optional[object] = None # Qwen3ASRModel or ForcedAlignerModel
|
||||
_align_model: Optional[object] = None # Qwen3-ForcedAlignerModel
|
||||
|
||||
# Model paths for loading
|
||||
MODEL_ID_HF = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
|
||||
MODEL_ID_MS = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
|
||||
ASR_MODEL_ID = os.getenv("ASR_MODEL_ID", "small")
|
||||
ASR_COMPUTE_TYPE = os.getenv("ASR_COMPUTE_TYPE", "int8")
|
||||
|
||||
# ModelScope ASR/ForcedAligner models (MLX 4-bit format)
|
||||
ASR_MODEL_ID_MS = "aufklarer/Qwen3-ASR-0.6B-MLX-4bit"
|
||||
ALIGN_MODEL_ID_MS = "aufklarer/Qwen3-ForcedAligner-0.6B-MLX"
|
||||
_tts_model: Optional["Qwen3TTSModel"] = None
|
||||
_asr_model: Optional["WhisperModel"] = None
|
||||
|
||||
|
||||
def _get_device_map() -> str:
|
||||
"""设备检测逻辑：优先 CUDA，其次 MPS，最后 CPU"""
|
||||
if torch is None:
|
||||
return "cpu"
|
||||
if torch.cuda.is_available():
|
||||
return "cuda:0"
|
||||
return "cuda"
|
||||
try:
|
||||
if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
|
||||
return "mps"
|
||||
except Exception as e: # noqa: ANN001
|
||||
logger.debug("MPS check failed: %s", e)
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.debug("MPS check failed: %s", exc)
|
||||
return "cpu"
|
||||
|
||||
|
||||
def _download_model_from_modelscope() -> Optional[str]:
|
||||
"""从 ModelScope 下载模型到本地缓存目录"""
|
||||
def _download_tts_model_from_modelscope() -> Optional[str]:
|
||||
if snapshot_download is None:
|
||||
return None
|
||||
cache_dir = os.path.join(os.path.dirname(__file__), "models")
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
try:
|
||||
cache_dir = os.path.join(os.path.dirname(__file__), "models")
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
model_dir = snapshot_download(
|
||||
MODEL_ID_MS,
|
||||
cache_dir=cache_dir,
|
||||
revision="master"
|
||||
)
|
||||
logger.info("ModelScope 模型下载完成: %s", model_dir)
|
||||
return model_dir
|
||||
except Exception as e: # noqa: ANN001
|
||||
logger.warning("ModelScope 下载失败: %s", e)
|
||||
return snapshot_download(MODEL_ID_MS, cache_dir=cache_dir, revision="master")
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.warning("ModelScope TTS download failed: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
async def _warmup_tts():
|
||||
"""预热 TTS 模型"""
|
||||
await asyncio.to_thread(_load_tts_model_with_retry)
|
||||
|
||||
|
||||
async def _warmup_asr():
|
||||
"""预热 ASR 模型（从 ModelScope 下载并加载）"""
|
||||
await asyncio.to_thread(_load_asr_models)
|
||||
|
||||
|
||||
async def _warmup_all():
|
||||
"""预热所有模型（TTS 和 ASR）"""
|
||||
logger.info("[Warmup] 开始预热 TTS 模型...")
|
||||
await _warmup_tts()
|
||||
logger.info("[Warmup] TTS 模型预热完成")
|
||||
|
||||
if Qwen3ASRModel is not None:
|
||||
logger.info("[Warmup] 开始预热 ASR 模型...")
|
||||
await _warmup_asr()
|
||||
logger.info("[Warmup] ASR 模型预热完成")
|
||||
|
||||
|
||||
def _load_tts_model_with_retry(max_retries: int = 3) -> "Qwen3TTSModel":
|
||||
"""加载 TTS 模型，支持多个镜像源"""
|
||||
def _ensure_tts_model() -> "Qwen3TTSModel":
|
||||
global _tts_model
|
||||
if _tts_model is not None:
|
||||
return _tts_model
|
||||
if Qwen3TTSModel is None:
|
||||
raise RuntimeError("qwen_tts 库未安装，无法加载 TTS 模型")
|
||||
if np is None or torch is None or Qwen3TTSModel is None:
|
||||
raise RuntimeError("TTS 依赖未安装完整")
|
||||
|
||||
device_map = _get_device_map()
|
||||
last_err = None
|
||||
dtype = torch.float16 if device_map != "cpu" else torch.float32
|
||||
|
||||
# 策略1: 尝试从 ModelScope 下载后加载
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
logger.info("尝试从 ModelScope 下载 TTS 模型...")
|
||||
model_path = _download_model_from_modelscope()
|
||||
if model_path and os.path.isdir(model_path):
|
||||
_tts_model = Qwen3TTSModel.from_pretrained( # type: ignore
|
||||
model_path,
|
||||
device_map=device_map,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
logger.info("ModelScope TTS 模型加载成功: %s", model_path)
|
||||
return _tts_model
|
||||
except Exception as e: # noqa: ANN001
|
||||
logger.warning("ModelScope TTS 加载失败 (尝试 %d/%d): %s", attempt + 1, max_retries, e)
|
||||
last_err = e
|
||||
model_path = _download_tts_model_from_modelscope()
|
||||
last_error = None
|
||||
|
||||
# 策略2: 尝试从 HuggingFace 镜像加载
|
||||
for attempt in range(max_retries):
|
||||
for candidate in [model_path, MODEL_ID_HF]:
|
||||
if not candidate:
|
||||
continue
|
||||
try:
|
||||
logger.info("尝试从 HuggingFace 镜像加载 TTS...")
|
||||
_tts_model = Qwen3TTSModel.from_pretrained( # type: ignore
|
||||
MODEL_ID_HF,
|
||||
candidate,
|
||||
device_map=device_map,
|
||||
dtype=torch.float16,
|
||||
dtype=dtype,
|
||||
)
|
||||
logger.info("HuggingFace TTS 模型加载成功")
|
||||
return _tts_model
|
||||
except Exception as e: # noqa: ANN001
|
||||
logger.warning("HuggingFace TTS 加载失败 (尝试 %d/%d): %s", attempt + 1, max_retries, e)
|
||||
last_err = e
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
logger.warning("TTS model load failed from %s: %s", candidate, exc)
|
||||
|
||||
raise RuntimeError(f"无法加载 TTS 模型: {last_err}") from last_err
|
||||
raise RuntimeError(f"TTS 模型加载失败: {last_error}") from last_error
|
||||
|
||||
|
||||
def _load_asr_models() -> None:
|
||||
"""从 ModelScope 下载并加载 ASR/ForcedAligner MLX 模型"""
|
||||
global _asr_model, _align_model
|
||||
|
||||
if snapshot_download is None:
|
||||
logger.warning("modelscope 未安装，跳过 ASR 模型加载")
|
||||
return
|
||||
|
||||
if Qwen3ASRModel is None:
|
||||
logger.warning("mlx_audio 未安装，跳过 ASR 模型加载")
|
||||
return
|
||||
|
||||
# Download and load ASR model from ModelScope
|
||||
try:
|
||||
logger.info("从 ModelScope 下载 ASR 模型...")
|
||||
asr_cache_dir = os.path.join(os.path.dirname(__file__), "models", "asr")
|
||||
asr_model_dir = snapshot_download(ASR_MODEL_ID_MS, cache_dir=asr_cache_dir)
|
||||
_load_asr_from_path(asr_model_dir)
|
||||
except Exception as e: # noqa: ANN001
|
||||
logger.warning("ASR ModelScope 下载失败，尝试 hf-mirror: %s", e)
|
||||
try:
|
||||
_load_asr_from_hf_mirror()
|
||||
except Exception as e2: # noqa: ANN001
|
||||
logger.warning("ASR hf-mirror 加载失败，跳过 ASR: %s", e2)
|
||||
|
||||
# Download and load ForcedAligner model from ModelScope
|
||||
try:
|
||||
logger.info("从 ModelScope 下载 ForcedAligner 模型...")
|
||||
align_cache_dir = os.path.join(os.path.dirname(__file__), "models", "aligner")
|
||||
align_model_dir = snapshot_download(ALIGN_MODEL_ID_MS, cache_dir=align_cache_dir)
|
||||
_load_align_from_path(align_model_dir)
|
||||
except Exception as e: # noqa: ANN001
|
||||
logger.warning("ForcedAligner ModelScope 下载失败，尝试 hf-mirror: %s", e)
|
||||
try:
|
||||
_load_align_from_hf_mirror()
|
||||
except Exception as e2: # noqa: ANN001
|
||||
logger.warning("ForcedAligner hf-mirror 加载失败，跳过: %s", e2)
|
||||
|
||||
|
||||
def _load_asr_from_path(model_dir: str) -> None:
|
||||
"""从本地路径加载 ASR MLX 模型"""
|
||||
def _ensure_asr_model() -> "WhisperModel":
|
||||
global _asr_model
|
||||
try:
|
||||
from mlx_audio.stt.utils import load as stt_load # type: ignore
|
||||
if _asr_model is not None:
|
||||
return _asr_model
|
||||
if WhisperModel is None:
|
||||
raise RuntimeError("faster-whisper 未安装")
|
||||
|
||||
model = stt_load(model_dir)
|
||||
_asr_model = model
|
||||
logger.info("ASR 模型加载成功 (路径: %s)", model_dir)
|
||||
except Exception as e: # noqa: ANN001
|
||||
logger.warning("ASR MLX 加载失败，尝试直接构建: %s", e)
|
||||
try:
|
||||
from mlx.core import load as mx_load # type: ignore
|
||||
|
||||
weights = mx_load(os.path.join(model_dir, "model.safetensors"))
|
||||
from mlx_lm import load as lm_load # type: ignore
|
||||
|
||||
model = lm_load(model_dir, model_cls=Qwen3ASRModel)
|
||||
_asr_model = model
|
||||
except Exception as e2: # noqa: ANN001
|
||||
raise RuntimeError(f"无法加载 ASR MLX 模型: {e2}") from e
|
||||
device = "cuda" if _get_device_map() == "cuda" else "cpu"
|
||||
compute_type = ASR_COMPUTE_TYPE if device == "cpu" else "float16"
|
||||
_asr_model = WhisperModel(ASR_MODEL_ID, device=device, compute_type=compute_type)
|
||||
return _asr_model
|
||||
|
||||
|
||||
def _load_asr_from_hf_mirror() -> None:
|
||||
"""从 hf-mirror 加载 ASR MLX 模型"""
|
||||
global _asr_model
|
||||
try:
|
||||
from mlx_audio.stt.utils import load as stt_load # type: ignore
|
||||
|
||||
model = stt_load("mlx-community/Qwen3-ASR-0.6B-4bit")
|
||||
_asr_model = model
|
||||
except Exception as e: # noqa: ANN001
|
||||
raise RuntimeError(f"无法从 hf-mirror 加载 ASR MLX: {e}") from e
|
||||
async def _warmup_tts():
|
||||
await asyncio.to_thread(_ensure_tts_model)
|
||||
|
||||
|
||||
def _load_align_from_path(model_dir: str) -> None:
|
||||
"""从本地路径加载 ForcedAligner MLX 模型"""
|
||||
global _align_model
|
||||
try:
|
||||
from mlx_audio.stt.utils import load as stt_load # type: ignore
|
||||
|
||||
model = stt_load(model_dir)
|
||||
_align_model = model
|
||||
except Exception as e: # noqa: ANN001
|
||||
raise RuntimeError(f"无法加载 ForcedAligner MLX 模型 (路径: {model_dir}): {e}") from e
|
||||
|
||||
|
||||
def _load_align_from_hf_mirror() -> None:
|
||||
"""从 hf-mirror 加载 ForcedAligner MLX 模型"""
|
||||
global _align_model
|
||||
try:
|
||||
from mlx_audio.stt.utils import load as stt_load # type: ignore
|
||||
|
||||
model = stt_load("mlx-community/Qwen3-ForcedAligner-0.6B-4bit")
|
||||
_align_model = model
|
||||
except Exception as e: # noqa: ANN001
|
||||
raise RuntimeError(f"无法从 hf-mirror 加载 ForcedAligner MLX: {e}") from e
|
||||
async def _warmup_asr():
|
||||
await asyncio.to_thread(_ensure_asr_model)
|
||||
|
||||
|
||||
class TTSRequest(BaseModel):
|
||||
@@ -286,43 +159,25 @@ class ModelStatus(BaseModel):
|
||||
device: str
|
||||
|
||||
|
||||
def _ensure_tts_model() -> "Qwen3TTSModel":
|
||||
"""确保 TTS 模型已加载"""
|
||||
global _tts_model
|
||||
if _tts_model is None:
|
||||
_tts_model = _load_tts_model_with_retry()
|
||||
return _tts_model
|
||||
|
||||
|
||||
def _ensure_asr_model():
|
||||
"""确保 ASR 模型已加载（懒加载）"""
|
||||
global _asr_model
|
||||
if _asr_model is None:
|
||||
try:
|
||||
from mlx_audio.stt.utils import load as stt_load # type: ignore
|
||||
|
||||
_asr_model = stt_load(ASR_MODEL_ID_MS)
|
||||
except Exception as e: # noqa: ANN001
|
||||
raise RuntimeError(f"无法加载 ASR MLX 模型 (路径: {ASR_MODEL_ID_MS}): {e}") from e
|
||||
return _asr_model
|
||||
|
||||
|
||||
def _ensure_align_model():
|
||||
"""确保 ForcedAligner 模型已加载（懒加载）"""
|
||||
global _align_model
|
||||
if _align_model is None:
|
||||
try:
|
||||
from mlx_audio.stt.utils import load as stt_load # type: ignore
|
||||
|
||||
_align_model = stt_load(ALIGN_MODEL_ID_MS)
|
||||
except Exception as e: # noqa: ANN001
|
||||
raise RuntimeError(f"无法加载 ForcedAligner MLX 模型 (路径: {ALIGN_MODEL_ID_MS}): {e}") from e
|
||||
return _align_model
|
||||
def _normalize_language(language: Optional[str]) -> Optional[str]:
|
||||
if not language:
|
||||
return None
|
||||
value = language.strip().lower()
|
||||
if value in {"auto", ""}:
|
||||
return None
|
||||
mapping = {
|
||||
"zh-cn": "zh",
|
||||
"zh-hans": "zh",
|
||||
"zh-tw": "zh",
|
||||
"en-us": "en",
|
||||
"ja-jp": "ja",
|
||||
"ko-kr": "ko",
|
||||
}
|
||||
return mapping.get(value, value.split("-")[0])
|
||||
|
||||
|
||||
@meta_router.get("/status", response_model=ModelStatus)
|
||||
async def get_status():
|
||||
"""获取模型状态"""
|
||||
return ModelStatus(
|
||||
tts_loaded=_tts_model is not None,
|
||||
asr_loaded=_asr_model is not None,
|
||||
@@ -332,11 +187,10 @@ async def get_status():
|
||||
|
||||
@meta_router.get("/config")
|
||||
async def get_config():
|
||||
"""获取配置信息"""
|
||||
return {
|
||||
"model": {
|
||||
"tts": MODEL_ID_MS,
|
||||
"asr": ASR_MODEL_ID_MS if Qwen3ASRModel is not None else None,
|
||||
"asr": ASR_MODEL_ID,
|
||||
},
|
||||
"device": _get_device_map(),
|
||||
"status": {
|
||||
@@ -348,15 +202,11 @@ async def get_config():
|
||||
|
||||
@meta_router.post("/warmup")
|
||||
async def warmup_models():
|
||||
"""手动触发模型预热"""
|
||||
await _warmup_tts()
|
||||
|
||||
if Qwen3ASRModel is not None:
|
||||
await _warmup_asr()
|
||||
|
||||
await _warmup_asr()
|
||||
return {
|
||||
"tts_warmup": _tts_model is not None,
|
||||
"asr_warmup": _asr_model is not None if Qwen3ASRModel else False,
|
||||
"asr_warmup": _asr_model is not None,
|
||||
"device": _get_device_map(),
|
||||
}
|
||||
|
||||
@@ -367,26 +217,30 @@ async def generate_tts_response(
|
||||
speaker: str = "Vivian",
|
||||
output_format: str = "wav",
|
||||
) -> TTSResponse:
|
||||
del speaker # current model path does not expose multi-speaker routing
|
||||
del output_format # current implementation always returns wav
|
||||
try:
|
||||
model = _ensure_tts_model()
|
||||
except Exception as e: # noqa: ANN001
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
del speaker
|
||||
del output_format
|
||||
if np is None:
|
||||
raise HTTPException(status_code=501, detail="numpy 未安装，TTS 功能不可用")
|
||||
|
||||
try:
|
||||
wavs, sr = model.generate_voice_design( # type: ignore
|
||||
model = _ensure_tts_model()
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc))
|
||||
|
||||
try:
|
||||
wavs, sample_rate = await asyncio.to_thread(
|
||||
model.generate_voice_design, # type: ignore
|
||||
text=text,
|
||||
language="Chinese",
|
||||
instruct=instruct or "",
|
||||
)
|
||||
except Exception as e: # noqa: ANN001
|
||||
logger.exception("TTS 推理失败")
|
||||
raise HTTPException(status_code=500, detail=f"TTS 推理失败: {e}")
|
||||
except Exception as exc:
|
||||
logger.exception("TTS inference failed")
|
||||
raise HTTPException(status_code=500, detail=f"TTS 推理失败: {exc}")
|
||||
|
||||
wav_data = wavs[0] if isinstance(wavs, (list, tuple)) else wavs
|
||||
if hasattr(wav_data, 'numpy'): # type: ignore
|
||||
wav_data = wav_data.cpu().numpy() # type: ignore
|
||||
if hasattr(wav_data, "cpu"):
|
||||
wav_data = wav_data.cpu().numpy()
|
||||
wav_data = np.asarray(wav_data, dtype=np.float32)
|
||||
|
||||
tmp_path = None
|
||||
@@ -394,81 +248,66 @@ async def generate_tts_response(
|
||||
import soundfile as sf # type: ignore
|
||||
|
||||
fd, tmp_path = tempfile.mkstemp(suffix=".wav")
|
||||
os.close(fd) # type: ignore
|
||||
sf.write(tmp_path, wav_data, sr)
|
||||
with open(tmp_path, "rb") as f: # noqa: SIM115
|
||||
audio_bytes = f.read()
|
||||
except Exception as e: # noqa: ANN001
|
||||
logger.exception("音频编码失败")
|
||||
raise HTTPException(status_code=500, detail=f"音频编码失败: {e}")
|
||||
os.close(fd)
|
||||
sf.write(tmp_path, wav_data, sample_rate)
|
||||
with open(tmp_path, "rb") as handle:
|
||||
audio_bytes = handle.read()
|
||||
except Exception as exc:
|
||||
logger.exception("TTS audio encode failed")
|
||||
raise HTTPException(status_code=500, detail=f"音频编码失败: {exc}")
|
||||
finally:
|
||||
if tmp_path and os.path.exists(tmp_path): # noqa: SIM201
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except Exception:
|
||||
pass
|
||||
if tmp_path and os.path.exists(tmp_path):
|
||||
os.unlink(tmp_path)
|
||||
|
||||
duration_ms = int(len(wav_data) / sr * 1000) if sr > 0 else 0
|
||||
audio_base64 = base64.b64encode(audio_bytes).decode("utf-8")
|
||||
return TTSResponse(audio_base64=audio_base64, format="wav", duration_ms=duration_ms)
|
||||
duration_ms = int(len(wav_data) / sample_rate * 1000) if sample_rate > 0 else 0
|
||||
return TTSResponse(
|
||||
audio_base64=base64.b64encode(audio_bytes).decode("utf-8"),
|
||||
format="wav",
|
||||
duration_ms=duration_ms,
|
||||
)
|
||||
|
||||
|
||||
async def generate_asr_response(audio_bytes: bytes, language: Optional[str] = "zh-CN") -> ASRResponse:
|
||||
if Qwen3ASRModel is None:
|
||||
raise HTTPException(status_code=501, detail="mlx_audio 未安装，ASR 功能不可用")
|
||||
if not audio_bytes:
|
||||
raise HTTPException(status_code=400, detail="音频内容为空")
|
||||
|
||||
try:
|
||||
model = _ensure_asr_model()
|
||||
except Exception as e: # noqa: ANN001
|
||||
raise HTTPException(status_code=500, detail=f"ASR 模型加载失败: {e}")
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=f"ASR 模型加载失败: {exc}")
|
||||
|
||||
normalized_language = _normalize_language(language)
|
||||
tmp_path = None
|
||||
try:
|
||||
wav_buffer = io.BytesIO(audio_bytes)
|
||||
with wave.open(wav_buffer, 'rb') as wf: # noqa: SIM115
|
||||
n_channels = wf.getnchannels()
|
||||
sampwidth = wf.getsampwidth()
|
||||
framerate = wf.getframerate()
|
||||
n_frames = wf.getnframes()
|
||||
fd, tmp_path = tempfile.mkstemp(suffix=".wav")
|
||||
os.close(fd)
|
||||
with open(tmp_path, "wb") as handle:
|
||||
handle.write(audio_bytes)
|
||||
|
||||
raw_data = wf.readframes(n_frames)
|
||||
audio_array = np.frombuffer(raw_data, dtype=np.int16 if sampwidth == 2 else np.float32)
|
||||
|
||||
if n_channels > 1:
|
||||
audio_array = np.mean(audio_array.reshape(-1, n_channels), axis=1)
|
||||
|
||||
if framerate != 16000:
|
||||
try:
|
||||
import scipy.signal as signal # type: ignore
|
||||
|
||||
n_samples = int(len(audio_array) * 16000 / framerate)
|
||||
audio_array = signal.resample(audio_array, n_samples) # type: ignore
|
||||
except Exception as e2: # noqa: ANN001
|
||||
logger.warning("重采样失败，使用原始音频: %s", e2)
|
||||
|
||||
if audio_array.dtype == np.int16:
|
||||
audio_array = audio_array.astype(np.float32) / 32768.0
|
||||
|
||||
result = model.generate( # type: ignore
|
||||
audio_array,
|
||||
language=language if language else None,
|
||||
segments, info = await asyncio.to_thread(
|
||||
model.transcribe,
|
||||
tmp_path,
|
||||
language=normalized_language,
|
||||
vad_filter=True,
|
||||
beam_size=5,
|
||||
)
|
||||
|
||||
recognized_text = getattr(result, 'text', str(result)) if hasattr(result, 'text') else str(result)
|
||||
detected_lang = getattr(result, 'language', language or "zh-CN")
|
||||
if isinstance(detected_lang, list) and len(detected_lang) > 0:
|
||||
detected_lang = detected_lang[0]
|
||||
|
||||
return ASRResponse(text=recognized_text, language=str(detected_lang))
|
||||
text = "".join(segment.text for segment in segments).strip()
|
||||
if not text:
|
||||
raise RuntimeError("ASR 返回结果为空")
|
||||
detected_language = getattr(info, "language", normalized_language or "unknown")
|
||||
return ASRResponse(text=text, language=str(detected_language))
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e: # noqa: ANN001
|
||||
logger.exception("ASR 推理失败")
|
||||
raise HTTPException(status_code=500, detail=f"ASR 推理失败: {e}")
|
||||
except Exception as exc:
|
||||
logger.exception("ASR inference failed")
|
||||
raise HTTPException(status_code=500, detail=f"ASR 推理失败: {exc}")
|
||||
finally:
|
||||
if tmp_path and os.path.exists(tmp_path):
|
||||
os.unlink(tmp_path)
|
||||
|
||||
|
||||
@generation_router.post("/tts", response_model=TTSResponse)
|
||||
async def tts_endpoint(req: TTSRequest):
|
||||
"""TTS 文字转语音端点"""
|
||||
return await generate_tts_response(
|
||||
text=req.text,
|
||||
instruct=req.instruct or "",
|
||||
@@ -479,18 +318,11 @@ async def tts_endpoint(req: TTSRequest):
|
||||
|
||||
@generation_router.post("/asr", response_model=ASRResponse)
|
||||
async def asr_endpoint(req: ASRRequest):
|
||||
"""语音识别端点（非流式）"""
|
||||
audio_bytes = base64.b64decode(req.audio_base64)
|
||||
return await generate_asr_response(audio_bytes, req.language if req.language else None)
|
||||
|
||||
|
||||
def register_tts_asr_routes(app, include_generation_routes: bool = True):
|
||||
"""注册 TTS/ASR 路由到 FastAPI 应用"""
|
||||
app.include_router(meta_router, prefix="/v1/tts-asr")
|
||||
if include_generation_routes:
|
||||
app.include_router(generation_router, prefix="/v1/tts-asr")
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(meta_router)
|
||||
router.include_router(generation_router)
|
||||
|
||||
Reference in New Issue
Block a user