From 23bfca51e49ecf550b9a6abdd00ef23e82a277bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cydy0615=E2=80=9D?= <“allenyuan410@gmail.com”> Date: Sat, 27 Jun 2026 22:22:42 +0800 Subject: [PATCH] feat: sync full-stack Docker runtime and UI --- .dockerignore | 6 + AGENTS.md | 8 +- CLAUDE.md | 105 --- Dockerfile.frontend | 3 +- README.md | 36 +- backend/.env.example | 45 +- backend/AGENTS.md | 6 +- backend/Dockerfile | 10 +- backend/audit_store.py | 35 +- backend/job_handlers.py | 233 +++++-- backend/job_system.py | 191 +++++- backend/llm.py | 16 +- backend/llm_policy.py | 20 + backend/main.py | 142 ++-- backend/requirements.docker.txt | 7 - backend/requirements.txt | 14 +- backend/risk_config.py | 22 +- backend/tests/TESTING_GUIDE.md | 453 ------------- backend/tests/benchmark_tts_asr.py | 209 ++++++ backend/tests/quick_verify.py | 188 ------ backend/tests/run_tests.py | 217 ++---- backend/tests/simulate_macos.py | 504 -------------- backend/tests/test_audit_store.py | 75 +++ backend/tests/test_main_cancel.py | 66 ++ backend/tests/test_main_endpoints.py | 11 + backend/tests/test_tts_asr.py | 334 +++++++++ backend/tests/test_web_search.py | 17 +- backend/tts_asr.py | 551 ++++++++------- docker-compose.yml | 71 +- docker/nginx.conf | 4 - src/App.vue | 3 +- src/components/DocBlockCrepe.vue | 34 +- src/components/FileContent.vue | 7 + src/components/FileTree.vue | 9 + src/components/HiddenTextCrepe.vue | 6 +- src/components/ImageEditorComponent.vue | 8 +- src/components/MarkdownPreview.vue | 6 +- src/components/MilkdownEditor.vue | 861 +++++++++++++++++++----- src/components/OfficePreview.vue | 5 +- src/components/ProBlockCrepe.vue | 5 +- src/components/SettingsPanel.vue | 29 + src/components/UploadBlockCrepe.vue | 3 +- src/components/VideoPlayer.vue | 4 +- src/components/WebSearchBlockCrepe.vue | 36 +- src/stores/settings.js | 1 - src/style.css | 178 ++++- src/utils/convert.js | 8 +- src/utils/inputBlock.js | 59 +- src/views/DocsView.vue | 6 +- src/views/EditorView.vue | 3 +- 50 files changed, 2750 insertions(+), 2120 deletions(-) delete mode 100644 CLAUDE.md delete mode 100644 backend/tests/TESTING_GUIDE.md create mode 100644 backend/tests/benchmark_tts_asr.py delete mode 100644 backend/tests/quick_verify.py delete mode 100644 backend/tests/simulate_macos.py create mode 100644 backend/tests/test_audit_store.py create mode 100644 backend/tests/test_tts_asr.py diff --git a/.dockerignore b/.dockerignore index 14aae47..254bc39 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,13 @@ node_modules dist htmlcov .pytest_cache +.coverage +.build-check +reports .git docker-data +backend/.env +backend/models backend/__pycache__ backend/tests/__pycache__ +**/.DS_Store diff --git a/AGENTS.md b/AGENTS.md index 32bdbfd..3e37a07 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ - 这是一个智能 Markdown 编辑器，前端负责编辑器 UI、上传导出、补全交互和设置状态，后端负责 LLM、OCR、文件转换和 TTS 接口。 - 前端技术栈：Vue 3 + Vite + Milkdown/Crepe + Pinia + Vue Router。 -- 后端技术栈：FastAPI + Python + Ollama-compatible LLM endpoint + Redis Streams。 +- 后端技术栈：FastAPI + Python + OpenAI-compatible LLM endpoint + Redis Streams。 - 项目版本：v0.2.0（自 b82c6d3 之后的全栈架构升级版本）。 ## 功能块系统（核心概念） @@ -81,12 +81,12 @@ - OCR 文本和文档块内容会被注入补全上下文，但这些内容属于隐藏上下文，不应被直接当作用户可见文本重复输出。 - /v1/convert 当前支持 txt、docx、pptx、pdf，非 txt 文件通过 MarkItDown 转成 Markdown，之后会清理图片标记。 - **AI 开关是全局广播状态**：MilkdownEditor.vue 通过 `llm-in-text:copilot-toggle` 同步主编辑器、文档块嵌套编辑器、网页搜索块嵌套编辑器；修 ghost text 时要同时检查这三处。 -- **设置项已从 currency 改为 country**：前后端请求、prompt、store、设置面板统一使用 `country`；仅在读取旧 localStorage 时兼容 `currency` 作为迁移兜底。 +- **设置项统一使用 country**：前后端请求、prompt、store、设置面板只使用 `country`。 - **DOCX/PDF 导出改为纯前端**：不再依赖 `/v1/export/pdf`。当前策略是先展开所有功能块，再从编辑器 HTML 构建导出内容；`src/utils/richExport.js` 负责 HTML -> PDF / DOCX。 - **上传单文件限制统一为 100MB**：前端校验和后端 OCR 风控上限都按 100MB 处理。 - **视频解析策略**：上传视频时，后端 `/v1/ocr` 接收 `media_type=video`，视频画面走 OCR 模型，音轨通过 ffmpeg 抽取后走 ASR 模型，最终合并为“视频画面 OCR + 视频音频 ASR”文本。 - **OCR 明确关闭思考**：backend/llm.py 的 OCR payload 显式下发 `options.think = False` 与 `temperature = 0`。 -- **TTS/ASR 当前真实实现**：backend/tts_asr.py 已切到 `Qwen3TTSModel + faster-whisper` 路线；不要继续按旧的 MLX-only 文档理解当前实现。 +- **TTS/ASR 当前真实实现**：backend/tts_asr.py 统一通过 `LLM_BASE_URL` + `LLM_API_KEY` 调用 OpenAI-compatible Speech API，默认模型为 `Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit` 与 `Qwen3-ASR-0.6B-8bit`。 ## 常用命令 @@ -124,7 +124,7 @@ 6. 再次用 `docker compose exec -T ...` 验证容器内文件和行为，不能只看本地文件。 - Docker 持久化数据统一落在部署目录内的 `docker-data/`，包括 PostgreSQL、Redis 和任务共享临时目录。 - 容器内访问宿主机模型服务时，不要继续使用 `localhost`；应改成 `host.docker.internal` 之类的容器可达地址。 -- 当前 Docker 部署的 `backend/requirements.docker.txt` 已包含 OCR、转换、队列以及 `torch` / `qwen-tts` / `faster-whisper`，并在 `backend/Dockerfile` 中额外安装 `ffmpeg` 以支持视频拆音轨。 +- 当前 Docker 部署的 `backend/requirements.docker.txt` 已包含 OCR、转换、队列和基础 API 依赖；`backend/Dockerfile` 额外安装 `ffmpeg` 以支持视频拆音轨。 - **Worker 容器**：worker.py 作为独立服务运行，通过 Redis Streams 消费任务队列。修改 job_handlers.py 或 worker.py 后需要验证 worker 容器内的代码已更新，可通过 `docker compose exec -T worker sh -lc "python -c 'from backend.job_handlers import get_handler; print(get_handler(\"completion\").__name__)'"` 验证。 - **Redis Streams 架构**：任务队列使用 Redis Streams，支持并发控制、速率限制和熔断器。job_system.py 定义 JOB_TYPES 和队列配置，worker.py 注册处理器并运行事件循环。 - 修改 Docker 相关文件时，除了代码本身，还要同步检查： diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 6641210..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,105 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with this repository. - -**Project version: v0.2.0** (full-stack architecture upgrade since b82c6d3) - -## Project Overview - -**LLM in Text** is an AI-powered Markdown editor built with Vue 3 + Vite (frontend) and FastAPI + Python + Ollama + Redis Streams (backend). It provides real-time AI completion suggestions, OCR image recognition, document conversion (PDF/DOCX/PPTX to Markdown), TTS text-to-speech, web search integration (via SearXNG + Firecrawl), CAPTCHA verification, and async job queue processing. - -- Completion interface uses plain POST/JSON (not SSE). Frontend sends `X-Request-Id` and calls `/v1/completions/cancel` on abort. -- AI completion is disabled when the document exceeds 32 KB (enforced both in UI and plugin layer). -- OCR text and doc-block content are injected into completion context as hidden context — they should NOT be rendered as visible user text. -- `/v1/convert` supports txt, docx, pptx, pdf via MarkItDown. Non-txt files go through Markdown sanitization (image removal, newline compression). -- `/v1/export/pdf` is called from the frontend but may not be implemented on the backend — verify before debugging PDF export. -- **Task queue architecture (v0.2.0 new)**: Backend shifted from synchronous endpoints to Redis Streams async queue. `job_system.py` defines JOB_TYPES (completion/pro_completion/web_search/compress/ocr/convert/tts/asr), `worker.py` consumes the queue, `job_handlers.py` registers handlers per type. All tasks support concurrency control, rate limiting, and circuit breakers. -- **Session tracking**: `session_store.py` provides InMemorySessionStore (dev) and PostgresSessionStore (prod), tracking request identity via session_hash + ip_hash. -- **Risk control system**: `risk_config.py` + `risk_control.py` implement rate limiting (sliding window), concurrency limits, circuit breaker pattern, and budget tracking — all thresholds via environment variables. - -## Quick Start - -```bash -# Frontend -npm install -npm run dev # Vite dev server on port 5173, proxies /v1 to backend - -# Backend -pip install -r backend/requirements.txt -python backend/main.py # port 8001 - -# Tests (90% coverage gate on backend modules) -pytest # full suite with coverage - -# Single test file (faster, no coverage overhead) -pytest backend/tests/test_prompt.py -v --no-cov - -# Build for production -npm run build -``` - -## Architecture - -### Frontend (`src/`) - -| Layer | Key Files | Responsibility | -|-------|-----------|----------------| -| Entry | `main.js`, `App.vue` | Vue app bootstrap, Pinia + Router mount | -| Routing | `router/index.js` | `/` → EditorView, `/docs` → DocsView | -| Editor | `components/MilkdownEditor.vue` | Central control: Crepe editor, plugin registration, upload/export/OCR/TTS/AI toggle, 32 KB limit | -| Plugins (TypeScript) | `plugins/copilotPlugin.ts` — ghost text, request scheduling, cancel, language detection, hidden context injection | -| Plugins (TypeScript) | `plugins/docBlockPlugin.ts` — doc-block nodes and rendering | -| Plugins (TypeScript) | `plugins/mermaidPlugin.ts` — Mermaid diagram preview | -| Store | `stores/settings.js` | localStorage-persisted settings (theme, modelThinking, debounceMs, privacyMode, language, background*, ttsInstruct) | -| API | `utils/api.js` — fetchSuggestion, cancel completion, TTS requests; `config.js` — VITE_* env-based URL config | -| Utilities | `utils/convert.js`, `ocrCache.js`, `docBlock.js`, `i18n.js` | - -### Backend (`backend/`) - -| File | Responsibility | -|------|----------------| -| `main.py` | FastAPI app, CORS, API key auth, routes: `/v1/completions`, `/v1/ocr`, `/v1/convert`, `/v1/completions/cancel`. TTS routes lazily registered from `tts_asr.py`. | -| `llm.py` | Async Ollama calls (`call_ollama`, `stream_ollama`) and VLM OCR (`call_vlm_ocr`). Timeout control. | -| `prompt.py` | Prompt assembly: `build_completion_prompts`, `prepare_prompt_context`. Templates from `prompts/` directory. | -| `pro_completions.py` | Pro-tier completion endpoint (newer addition). | -| `tts_asr.py` | TTS text-to-speech. Late-registered routes via `_register_tts_asr_routes`. | -| `geoip.py` | Client IP location lookup for non-privacy-mode requests. | - -### Request Flow: Completion - -``` -MilkdownEditor.vue → copilotPlugin.ts (debounce, abort, language detection) - → utils/api.js (fetchSuggestion: generates request_id, AbortSignal, reads settings) - → backend/main.py (/v1/completions: auth, prompt context, call_ollama via asyncio.Task) - → backend/prompt.py (system + user prompt from prefix/suffix/context) - → backend/llm.py (call_ollama to Ollama) - ← JSON { content, request_id } - → copilotPlugin.ts (insertGhostText into editor) -``` - -## Debugging Paths - -| Issue | Trace Order | -|-------|-------------| -| Completion not firing | `MilkdownEditor.vue` → `copilotPlugin.ts` (check enabled, size limit, debounce) | -| Wrong completion result | `prompt.py` → `llm.py`. Check prompt context and language detection. | -| Cancel not working | `main.py` request_id lifecycle ↔ frontend `X-Request-Id` + cancel call | -| OCR empty result | `main.py` base64 decode → `llm.py call_vlm_ocr` | -| Document conversion dirty | `_sanitize_converted_markdown` in `main.py` | - -## Naming Conventions (Mixed) - -- Vue components/views: PascalCase (`MilkdownEditor.vue`) -- Frontend utils/config: lowercase `.js` (`api.js`, `config.js`) -- Plugin layer: TypeScript (`.ts`) -- Python backend: snake_case - -Follow the style of each file. Do not reformat across directories for consistency. UI copy defaults to Chinese. - -## Important Rules - -- Do not modify `milkdown-docs/` (read-only reference). -- Code and tests override README.md when they conflict — the README is partially outdated. -- Plugin code (`copilotPlugin.ts`) is state-machine-style: small changes can break subtle interactions. Change one thing at a time and verify in-browser. -- No hardcoded secrets, empty catch/except blocks, `as any`, or `@ts-ignore` in new code. -- Subdirectory AGENTS.md files contain more detailed guidance: `./AGENTS.md` (root), `backend/AGENTS.md`, `src/AGENTS.md`, `src/plugins/AGENTS.md`. Read them when working in those areas. diff --git a/Dockerfile.frontend b/Dockerfile.frontend index 284bd42..5b3ecba 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -4,7 +4,8 @@ FROM ${DOCKER_REGISTRY_PREFIX}node:22-alpine AS build WORKDIR /app COPY package.json package-lock.json ./ -RUN npm ci +RUN --mount=type=cache,target=/root/.npm \ + npm ci --prefer-offline --no-audit COPY index.html vite.config.js ./ COPY public ./public diff --git a/README.md b/README.md index 4ac176a..0926fd1 100644 --- a/README.md +++ b/README.md @@ -106,9 +106,8 @@ docker compose up -d --build - PUT /v1/docs/files/{id}/blob 替换文件二进制内容 - DELETE /v1/docs/nodes/{id} 删除节点 - GET /v1/docs/files/{id}/blob 下载或预览原文件 -- GET /v1/tts-asr/status TTS/ASR模型状态 +- GET /v1/tts-asr/status TTS/ASR状态 - GET /v1/tts-asr/config TTS/ASR配置信息 -- POST /v1/tts-asr/warmup 模型预热 - POST /v1/tts-asr/tts 文字转语音 - POST /v1/tts-asr/asr 语音转文字 @@ -118,20 +117,14 @@ docker compose up -d --build | 变量名 | 说明 | 默认值 | |--------|------|--------| -| `TTS_ASR_DEVICE` | 设备选择 (auto/mps/cuda/cpu) | auto | -| `TTS_ASR_MODEL_SIZE` | ASR模型大小 (tiny/base/small/medium/large/turbo) | auto | -| `TTS_ASR_QUANTIZE` | 是否使用INT8量化 (true/false) | false | -| `TTS_ASR_OFFLINE_MODE` | 离线模式，仅使用缓存模型 (true/false) | false | -| `TTS_ASR_WARMUP` | 启动时预热模型 (true/false) | true | -| `TTS_ASR_WARMUP_TIMEOUT` | 预热超时时间(秒) | 120 | -| `TTS_ASR_IDLE_TIMEOUT` | 空闲卸载时间(秒，0=不卸载) | 0 | -| `TTS_ASR_MPS_MEMORY_LIMIT_MB` | MPS内存限制(MB) | 8192 | - -**Apple Silicon优化建议**: -- 系统自动检测Apple Silicon并推荐使用`small`模型 -- MPS内存限制默认为系统内存的60% -- 建议使用`small`或`medium`模型以获得更好的性能 -- 可通过`TTS_ASR_MODEL_SIZE=medium`手动指定模型大小 +| `LLM_BASE_URL` | OpenAI-compatible 上游地址 | 必填 | +| `LLM_API_KEY` | OpenAI-compatible 上游密钥 | 必填 | +| `TTS_MODEL_ID` | TTS 模型名 | `Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit` | +| `ASR_MODEL_ID` | ASR 模型名 | `Qwen3-ASR-0.6B-8bit` | +| `TTS_ASR_TTS_TIMEOUT_SECONDS` | TTS 上游超时(秒) | 180 | +| `TTS_ASR_ASR_TIMEOUT_SECONDS` | ASR 上游超时(秒) | 300 | +| `TTS_ASR_MAX_CONNECTIONS` | Speech API 连接池上限 | 24 | +| `TTS_ASR_MAX_KEEPALIVE_CONNECTIONS` | Speech API keepalive 连接数 | 12 | ## 核心实现 @@ -139,13 +132,10 @@ docker compose up -d --build - main.py: FastAPI服务器、SSE流式响应 - llm.py: 异步LLM调用（OpenAI兼容）、超时控制 - prompt.py: 7条Prompt规则 -- tts_asr.py: macOS/Apple Silicon优化的TTS/ASR处理 - - 自动检测Apple Silicon (M1/M2/M3) - - MPS/CUDA/CPU智能降级 - - 支持多种Whisper模型大小 - - INT8量化支持 - - 离线模式支持 - - 健壮的音频重采样 +- tts_asr.py: 基于共享 OpenAI-compatible Speech API 的 TTS/ASR 适配层 + - 统一使用 `LLM_BASE_URL` 和 `LLM_API_KEY` + - 通过 `/audio/speech` 与 `/audio/transcriptions` 调用上游 + - 内建连接池、超时、音频时长估算和上游请求 ID 透传 ### 前端 - copilotPlugin.ts: ProseMirror Mark系统 diff --git a/backend/.env.example b/backend/.env.example index f52c5b1..54d4f6d 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -4,13 +4,13 @@ LLM_BASE_URL=https://api.openai.com/v1/ LLM_API_KEY=sk-your-key # Default model for inline completions -LLM_MODEL=gpt-4.1-mini +LLM_MODEL=Nex-N2-mini-mlx-OptiQ-8bit-MTP # Pro-tier model (defaults to LLM_MODEL if unset) -PRO_LLM_MODEL=gpt-4.1 +PRO_LLM_MODEL=Nex-N2-mini-mlx-OptiQ-8bit-MTP # Vision model for OCR -VLM_MODEL=gpt-4.1-mini +VLM_MODEL=Nex-N2-mini-mlx-OptiQ-8bit-MTP # API key for the FastAPI app (change in production) API_KEY=your-secret-key-here @@ -56,10 +56,10 @@ JOB_OCR_CONCURRENCY=1 JOB_OCR_MAX_QUEUE=8 JOB_CONVERT_CONCURRENCY=1 JOB_CONVERT_MAX_QUEUE=8 -JOB_TTS_CONCURRENCY=1 -JOB_TTS_MAX_QUEUE=4 -JOB_ASR_CONCURRENCY=1 -JOB_ASR_MAX_QUEUE=4 +JOB_TTS_CONCURRENCY=4 +JOB_TTS_MAX_QUEUE=16 +JOB_ASR_CONCURRENCY=2 +JOB_ASR_MAX_QUEUE=8 # Timeouts (seconds) LLM_COMPLETION_TIMEOUT=600 @@ -88,10 +88,12 @@ RISK_MODEL_CIRCUIT_TTL_SECONDS=300 RISK_ENFORCE_REDIS_FAIL_CLOSED=false # Backend-controlled model policy -RISK_COMPLETION_MODEL=gpt-4.1-mini -RISK_PRO_MODEL=gpt-4.1 -RISK_VISION_MODEL=gpt-4.1-mini -RISK_WEB_SEARCH_MODEL=gpt-4.1-mini +RISK_COMPLETION_MODEL=Nex-N2-mini-mlx-OptiQ-8bit-MTP +RISK_PRO_MODEL=Nex-N2-mini-mlx-OptiQ-8bit-MTP +RISK_VISION_MODEL=Nex-N2-mini-mlx-OptiQ-8bit-MTP +RISK_WEB_SEARCH_MODEL=Nex-N2-mini-mlx-OptiQ-8bit-MTP +RISK_SPEECH_TTS_MODEL=Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit +RISK_SPEECH_ASR_MODEL=Qwen3-ASR-0.6B-8bit RISK_COMPLETION_MAX_INPUT_CHARS=24000 RISK_COMPLETION_MAX_OUTPUT_TOKENS=768 RISK_COMPLETION_TEMPERATURE=0.4 @@ -104,6 +106,8 @@ RISK_WEB_SEARCH_TEMPERATURE=0.4 RISK_COMPRESS_MAX_INPUT_CHARS=128000 RISK_COMPRESS_MAX_OUTPUT_TOKENS=1536 RISK_OCR_MAX_INPUT_BYTES=104857600 +RISK_SPEECH_TTS_MAX_INPUT_CHARS=4096 +RISK_SPEECH_ASR_MAX_INPUT_BYTES=104857600 # Web search providers SEARXNG_BASE_URL=http://searxng:8080 @@ -120,9 +124,18 @@ RISK_PRO_INPUT_COST_PER_1K=0.003 RISK_PRO_OUTPUT_COST_PER_1K=0.012 RISK_VISION_INPUT_COST_PER_1K=0.0008 RISK_VISION_OUTPUT_COST_PER_1K=0.0024 +RISK_SPEECH_TTS_INPUT_COST_PER_1K_CHARS=0 +RISK_SPEECH_TTS_OUTPUT_COST_PER_MINUTE_AUDIO=0 +RISK_SPEECH_ASR_INPUT_COST_PER_MB=0 -# Legacy fallback: if LLM_BASE_URL is not set, OLLAMA_HOST will be auto-converted to /v1/ path -#OLLAMA_HOST=http://localhost:11434 - -# TTS/ASR settings (see README for full list) -TTS_ASR_DEVICE=auto +# Shared speech API settings (uses LLM_BASE_URL + LLM_API_KEY) +TTS_MODEL_ID=Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit +TTS_DEFAULT_INSTRUCTIONS=A clear, natural voice speaking Mandarin Chinese. +ASR_MODEL_ID=Qwen3-ASR-0.6B-8bit +TTS_ASR_MAX_TEXT_CHARS=4096 +ASR_MAX_AUDIO_BYTES=104857600 +TTS_ASR_TTS_TIMEOUT_SECONDS=180 +TTS_ASR_ASR_TIMEOUT_SECONDS=300 +TTS_ASR_HEALTHCHECK_TIMEOUT_SECONDS=5 +TTS_ASR_MAX_CONNECTIONS=24 +TTS_ASR_MAX_KEEPALIVE_CONNECTIONS=12 diff --git a/backend/AGENTS.md b/backend/AGENTS.md index fd93f4a..80000cf 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -5,9 +5,9 @@ ## 后端职责 - 对外提供补全、取消补全、OCR、文档转换和 TTS/ASR 相关接口。 -- 组织 Prompt，上下文清洗，调用 Ollama 模型。 +- 组织 Prompt，上下文清洗，调用 OpenAI-compatible 模型接口。 - **通过 Redis Streams 异步任务队列处理各类作业（completion/PRO/web_search/compress/OCR/convert/TTS/ASR）。** -- 负责 API Key 校验、日志记录和部分启动预热逻辑。 +- 负责 API Key 校验、日志记录和队列任务路由。 ## 先看哪里 @@ -109,7 +109,7 @@ - 通过 _register_tts_asr_routes 延迟导入并挂到主应用。 - **TTS 请求通过 job_handlers.py tts_handler 处理。** - **ASR 请求通过 job_handlers.py asr_handler 处理。** -- **当前实现是 `Qwen3TTSModel + faster-whisper`，不是旧的 edge-tts / macos-say / MLX-only 路线。** +- **当前实现统一通过 `LLM_BASE_URL` + `LLM_API_KEY` 调用共享 Speech API，默认模型为 `Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit` 和 `Qwen3-ASR-0.6B-8bit`。** ## 开发命令 diff --git a/backend/Dockerfile b/backend/Dockerfile index a144e47..492bd56 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -6,12 +6,14 @@ ENV PYTHONUNBUFFERED=1 WORKDIR /app/backend -RUN apt-get update \ - && apt-get install -y --no-install-recommends ffmpeg \ - && rm -rf /var/lib/apt/lists/* +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ + apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg COPY backend/requirements.docker.txt /tmp/requirements.docker.txt -RUN pip install --no-cache-dir -r /tmp/requirements.docker.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install -r /tmp/requirements.docker.txt COPY backend /app/backend diff --git a/backend/audit_store.py b/backend/audit_store.py index 05b6f2f..e7c5b6f 100644 --- a/backend/audit_store.py +++ b/backend/audit_store.py @@ -76,6 +76,9 @@ class PostgresAuditStore(BaseAuditStore): estimated_cost NUMERIC(18, 8) NOT NULL DEFAULT 0, actual_output_chars INTEGER NOT NULL DEFAULT 0, actual_cost NUMERIC(18, 8) NOT NULL DEFAULT 0, + queue_ms INTEGER NOT NULL DEFAULT 0, + run_ms INTEGER NOT NULL DEFAULT 0, + total_ms INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL, error_code TEXT NOT NULL DEFAULT '', started_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -84,6 +87,15 @@ class PostgresAuditStore(BaseAuditStore): ) """ ) + cur.execute( + "ALTER TABLE llm_call_audit ADD COLUMN IF NOT EXISTS queue_ms INTEGER NOT NULL DEFAULT 0" + ) + cur.execute( + "ALTER TABLE llm_call_audit ADD COLUMN IF NOT EXISTS run_ms INTEGER NOT NULL DEFAULT 0" + ) + cur.execute( + "ALTER TABLE llm_call_audit ADD COLUMN IF NOT EXISTS total_ms INTEGER NOT NULL DEFAULT 0" + ) cur.execute( """ CREATE TABLE IF NOT EXISTS risk_events ( @@ -112,6 +124,21 @@ class PostgresAuditStore(BaseAuditStore): ) """ ) + cur.execute( + "CREATE INDEX IF NOT EXISTS idx_api_request_audit_request_id ON api_request_audit (request_id)" + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS idx_api_request_audit_route_created_at ON api_request_audit (route, created_at DESC)" + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS idx_llm_call_audit_request_id ON llm_call_audit (request_id)" + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS idx_llm_call_audit_job_type_started_at ON llm_call_audit (job_type, started_at DESC)" + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS idx_llm_call_audit_model_started_at ON llm_call_audit (model, started_at DESC)" + ) self._initialized = True def record_api_request(self, payload: dict[str, Any]) -> None: @@ -152,9 +179,10 @@ class PostgresAuditStore(BaseAuditStore): INSERT INTO llm_call_audit ( request_id, session_hash, ip_hash, job_type, model, estimated_input_tokens, max_output_tokens, estimated_cost, - actual_output_chars, actual_cost, status, error_code, metadata_json + actual_output_chars, actual_cost, queue_ms, run_ms, total_ms, + status, error_code, metadata_json ) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb) """, ( payload["request_id"], @@ -167,6 +195,9 @@ class PostgresAuditStore(BaseAuditStore): float(payload.get("estimated_cost", 0.0)), int(payload.get("actual_output_chars", 0)), float(payload.get("actual_cost", 0.0)), + int(payload.get("queue_ms", 0)), + int(payload.get("run_ms", 0)), + int(payload.get("total_ms", 0)), payload["status"], payload.get("error_code", ""), json.dumps(metadata, ensure_ascii=False), diff --git a/backend/job_handlers.py b/backend/job_handlers.py index 564bc9e..f45e873 100644 --- a/backend/job_handlers.py +++ b/backend/job_handlers.py @@ -1,8 +1,12 @@ import asyncio +import io import ipaddress import json import os import re +import socket +import time +import zipfile from contextlib import suppress from datetime import datetime from typing import Any, Callable, Awaitable @@ -22,24 +26,19 @@ from prompt import ( ) from risk_config import load_risk_config from risk_control import RiskIdentity, estimate_tokens, get_risk_controller - -try: # pragma: no cover - optional heavy dependency path - from tts_asr import generate_asr_response, generate_tts_response -except Exception: # pragma: no cover - generate_tts_response = None - generate_asr_response = None +from tts_asr import generate_asr_response, generate_tts_response IMAGE_MARKDOWN_RE = re.compile(r"!\[[^\]]*]\([^)]+\)") IMAGE_HTML_RE = re.compile(r"]*>", re.IGNORECASE) ALLOWED_CONVERT_EXTENSIONS = {".txt", ".docx", ".pptx", ".pdf"} -SEARXNG_BASE_URL = (os.getenv("SEARXNG_BASE_URL", "http://searxng:8080") or "http://searxng:8080").rstrip("/") -SEARXNG_RESULT_LIMIT = max(1, int(os.getenv("SEARXNG_RESULT_LIMIT", "10") or "10")) -FIRECRAWL_BASE_URL = (os.getenv("FIRECRAWL_BASE_URL", "http://firecrawl:3002") or "http://firecrawl:3002").rstrip("/") -FIRECRAWL_API_KEY = os.getenv("FIRECRAWL_API_KEY", "").strip() -WEB_SEARCH_QUERY_COUNT = max(3, min(5, int(os.getenv("WEB_SEARCH_QUERY_COUNT", "4") or "4"))) -WEB_SEARCH_SELECTED_URL_LIMIT = max(5, min(20, int(os.getenv("WEB_SEARCH_SELECTED_URL_LIMIT", "10") or "10"))) -WEB_SEARCH_CRAWL_CONCURRENCY = max(1, min(5, int(os.getenv("WEB_SEARCH_CRAWL_CONCURRENCY", "3") or "3"))) +SEARXNG_BASE_URL = os.getenv("SEARXNG_BASE_URL", "http://searxng:8080").rstrip("/") +SEARXNG_RESULT_LIMIT = int(os.getenv("SEARXNG_RESULT_LIMIT", "10") or "10") +FIRECRAWL_BASE_URL = os.getenv("FIRECRAWL_BASE_URL", "http://firecrawl:3002").rstrip("/") +FIRECRAWL_API_KEY = os.getenv("FIRECRAWL_API_KEY", "").strip() or "" +WEB_SEARCH_QUERY_COUNT = int(os.getenv("WEB_SEARCH_QUERY_COUNT", "4") or "4") +WEB_SEARCH_SELECTED_URL_LIMIT = int(os.getenv("WEB_SEARCH_SELECTED_URL_LIMIT", "10") or "10") +WEB_SEARCH_CRAWL_CONCURRENCY = max(1, min(6, int(os.getenv("WEB_SEARCH_CRAWL_CONCURRENCY", "3") or "3"))) WEB_SEARCH_CRAWL_TIMEOUT_SECONDS = max(10, min(90, int(os.getenv("WEB_SEARCH_CRAWL_TIMEOUT_SECONDS", "35") or "35"))) _markitdown_instance = None _risk_config = load_risk_config() @@ -82,6 +81,52 @@ def _normalize_multiline_text(value: str) -> str: return (value or "").replace("\r\n", "\n").replace("\r", "\n").strip() +def _looks_like_text(raw_bytes: bytes) -> bool: + sample = raw_bytes[:8192] + if not sample or b"\x00" in sample: + return False + try: + text = sample.decode("utf-8") + except UnicodeDecodeError: + return False + if not text.strip(): + return False + control_count = sum( + 1 + for char in text + if (ord(char) < 32 and char not in "\t\n\r") or ord(char) == 127 + ) + return control_count / max(len(text), 1) < 0.05 + + +def _infer_convert_suffix(raw_bytes: bytes, filename: str) -> str: + sample = raw_bytes[:1024 * 1024] + if sample.startswith(b"%PDF-"): + return ".pdf" + if sample.startswith((b"PK\x03\x04", b"PK\x05\x06")): + try: + with zipfile.ZipFile(io.BytesIO(raw_bytes)) as archive: + names = set(archive.namelist()) + if any(name.startswith("ppt/") for name in names): + return ".pptx" + if any(name.startswith("word/") for name in names): + return ".docx" + except Exception: + pass + if _looks_like_text(sample): + return ".txt" + return "" + + +def _resolve_url_addresses(url: str) -> list[tuple[Any, ...]]: + parsed = urlparse((url or "").strip()) + host = (parsed.hostname or "").strip().lower() + if not host: + return [] + port = parsed.port or (443 if parsed.scheme == "https" else 80) + return socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + + def _is_blocked_public_url(url: str) -> bool: try: parsed = urlparse((url or "").strip()) @@ -92,13 +137,26 @@ def _is_blocked_public_url(url: str) -> bool: host = (parsed.hostname or "").strip().lower() if not host: return True - if host in {"localhost", "127.0.0.1", "::1"} or host.endswith(".local"): + if host in {"localhost", "127.0.0.1", "::1"} or host.endswith((".local", ".localhost")): return True try: ip = ipaddress.ip_address(host) - return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast + return not ip.is_global except ValueError: - return False + pass + try: + addresses = _resolve_url_addresses(url) + except Exception: + return True + for info in addresses: + address = info[4][0] + try: + ip = ipaddress.ip_address(address) + except ValueError: + continue + if not ip.is_global: + return True + return False def _strip_code_fence(value: str) -> str: @@ -239,7 +297,7 @@ async def _searxng_search(query: str, *, limit: int) -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] for item in payload.get("results") or []: url = str(item.get("url") or item.get("link") or "").strip() - if not url or _is_blocked_public_url(url): + if not url or await asyncio.to_thread(_is_blocked_public_url, url): continue results.append({ "title": str(item.get("title") or "").strip(), @@ -347,6 +405,7 @@ async def _exit_llm_execution( status: str, actual_output_text: str = "", error_code: str = "", + audit_metadata: dict[str, Any] | None = None, ) -> None: policy = (risk.get("policy") or {}) controller = get_risk_controller(_risk_config) @@ -361,11 +420,35 @@ async def _exit_llm_execution( "vision": _risk_config.vision_output_cost_per_1k, }.get(profile, _risk_config.completion_output_cost_per_1k) actual_output_tokens = estimate_tokens(actual_output_text) - actual_cost = round((estimated_input_tokens / 1000.0) * { - "completion": _risk_config.completion_input_cost_per_1k, - "pro": _risk_config.pro_input_cost_per_1k, - "vision": _risk_config.vision_input_cost_per_1k, - }.get(profile, _risk_config.completion_input_cost_per_1k) + (actual_output_tokens / 1000.0) * pricing_out, 8) + extra_metadata = dict(audit_metadata or {}) + if profile == "speech_tts": + actual_cost = round( + (int(extra_metadata.get("text_chars", 0) or 0) / 1000.0) * _risk_config.speech_tts_input_cost_per_1k_chars + + (int(extra_metadata.get("duration_ms", 0) or 0) / 60000.0) * _risk_config.speech_tts_output_cost_per_minute_audio, + 8, + ) + elif profile == "speech_asr": + actual_cost = round( + (int(extra_metadata.get("audio_bytes", 0) or 0) / (1024.0 * 1024.0)) * _risk_config.speech_asr_input_cost_per_mb, + 8, + ) + else: + actual_cost = round((estimated_input_tokens / 1000.0) * { + "completion": _risk_config.completion_input_cost_per_1k, + "pro": _risk_config.pro_input_cost_per_1k, + "vision": _risk_config.vision_input_cost_per_1k, + }.get(profile, _risk_config.completion_input_cost_per_1k) + (actual_output_tokens / 1000.0) * pricing_out, 8) + job_context = payload.get("job_context") or {} + now_ms = int(time.time() * 1000) + started_at = int(job_context.get("started_at", 0) or 0) + created_at = int(job_context.get("created_at", 0) or 0) + queue_ms = int(job_context.get("queue_ms", 0) or 0) + run_ms = int(job_context.get("run_ms", 0) or 0) + total_ms = int(job_context.get("total_ms", 0) or 0) + if not run_ms and started_at: + run_ms = max(0, now_ms - started_at) + if not total_ms: + total_ms = max(0, now_ms - created_at) if created_at else run_ms await asyncio.to_thread( store.record_llm_call, { @@ -381,7 +464,10 @@ async def _exit_llm_execution( "actual_cost": actual_cost, "status": status, "error_code": error_code, - "metadata": {"profile": profile}, + "queue_ms": queue_ms, + "run_ms": run_ms, + "total_ms": total_ms, + "metadata": {"profile": profile, **extra_metadata}, }, ) @@ -748,14 +834,13 @@ async def ocr_handler( if media_type == "video" or is_video_filename(filename, mime_type): asr_text = "" - if generate_asr_response is not None: - try: - await emit("progress", {"phase": "asr", "media_type": media_type}) - audio_bytes = await asyncio.to_thread(extract_audio_wav_bytes, path) - asr_response = await generate_asr_response(audio_bytes, language) - asr_text = getattr(asr_response, "text", "") or "" - except Exception as exc: - asr_text = f"(音频解析失败: {exc})" + try: + await emit("progress", {"phase": "asr", "media_type": media_type}) + audio_bytes = await asyncio.to_thread(extract_audio_wav_bytes, path) + asr_response = await generate_asr_response(audio_bytes, language) + asr_text = getattr(asr_response, "text", "") or "" + except Exception as exc: + raise RuntimeError(f"音频解析失败: {exc}") from exc if ocr_text.strip() or asr_text.strip(): text_parts = [] if ocr_text.strip(): @@ -792,12 +877,15 @@ async def convert_handler( ) -> dict[str, Any]: path = payload["input_path"] filename = payload.get("filename", "document") - ext = os.path.splitext(filename)[1].lower() - if ext not in ALLOWED_CONVERT_EXTENSIONS: + try: + temp_ext = os.path.splitext(path)[1].lower() + except Exception: + temp_ext = "" + if temp_ext not in ALLOWED_CONVERT_EXTENSIONS: _safe_unlink(path) raise ValueError("仅支持 txt、docx、pptx、pdf 格式") try: - if ext == ".txt": + if temp_ext == ".txt": with open(path, "rb") as handle: markdown = _sanitize_converted_markdown(handle.read().decode("utf-8", errors="ignore")) else: @@ -817,19 +905,45 @@ async def tts_handler( emit: Callable[[str, dict[str, Any]], Awaitable[None]], is_cancelled: Callable[[], bool], ) -> dict[str, Any]: - if generate_tts_response is None: - raise RuntimeError("TTS 功能当前不可用") - response = await generate_tts_response( - text=payload["text"], - instruct=payload.get("instruct", ""), - speaker=payload.get("speaker", "Vivian"), - output_format=payload.get("format", "wav"), - ) - if is_cancelled(): - raise asyncio.CancelledError() - result = response.dict() - await emit("result", result) - return result + text = str(payload.get("text", "") or "").strip() + if not text: + raise ValueError("TTS 文本为空") + identity, risk, lock_keys = await _enter_llm_execution(payload, emit) + try: + response = await generate_tts_response( + text=text, + instruct=str(payload.get("instruct", "") or ""), + speaker=str(payload.get("speaker", "Vivian") or "Vivian"), + output_format=str(payload.get("format", "wav") or "wav"), + ) + if is_cancelled(): + raise asyncio.CancelledError() + + result = dict(response) + await emit("result", result) + await _exit_llm_execution( + payload, + identity, + risk, + lock_keys, + status="completed", + audit_metadata={ + "speaker": result.get("speaker", ""), + "format": result.get("format", ""), + "duration_ms": int(result.get("duration_ms", 0) or 0), + "audio_bytes": int(result.get("audio_bytes", 0) or 0), + "text_chars": int(result.get("text_chars", len(text)) or len(text)), + "request_ms": int(result.get("request_ms", 0) or 0), + "upstream_request_id": result.get("upstream_request_id", ""), + }, + ) + return result + except asyncio.CancelledError: + await _exit_llm_execution(payload, identity, risk, lock_keys, status="cancelled", error_code="cancelled") + raise + except Exception: + await _exit_llm_execution(payload, identity, risk, lock_keys, status="failed", error_code="tts_failed") + raise async def asr_handler( @@ -837,17 +951,36 @@ async def asr_handler( emit: Callable[[str, dict[str, Any]], Awaitable[None]], is_cancelled: Callable[[], bool], ) -> dict[str, Any]: - if generate_asr_response is None: - raise RuntimeError("ASR 功能当前不可用") path = payload["input_path"] + identity, risk, lock_keys = await _enter_llm_execution(payload, emit) try: with open(path, "rb") as handle: audio_bytes = handle.read() response = await generate_asr_response(audio_bytes, payload.get("language", "zh-CN")) if is_cancelled(): raise asyncio.CancelledError() - result = response.dict() + result = dict(response) await emit("result", result) + await _exit_llm_execution( + payload, + identity, + risk, + lock_keys, + status="completed", + actual_output_text=result.get("text", "") or "", + audit_metadata={ + "language": result.get("language", ""), + "audio_bytes": int(result.get("audio_bytes", len(audio_bytes)) or len(audio_bytes)), + "request_ms": int(result.get("request_ms", 0) or 0), + "upstream_request_id": result.get("upstream_request_id", ""), + }, + ) return result + except asyncio.CancelledError: + await _exit_llm_execution(payload, identity, risk, lock_keys, status="cancelled", error_code="cancelled") + raise + except Exception: + await _exit_llm_execution(payload, identity, risk, lock_keys, status="failed", error_code="asr_failed") + raise finally: _safe_unlink(path) diff --git a/backend/job_system.py b/backend/job_system.py index 640e04a..17d2eb4 100644 --- a/backend/job_system.py +++ b/backend/job_system.py @@ -33,6 +33,14 @@ JOB_TYPES = ( "asr", ) + +def _int_env(name: str, default: int) -> int: + try: + return max(1, int(os.getenv(name, str(default)))) + except (TypeError, ValueError): + return default + + DEFAULT_CONCURRENCY = { "completion": 2, "pro_completion": 1, @@ -40,8 +48,8 @@ DEFAULT_CONCURRENCY = { "compress": 1, "ocr": 1, "convert": 1, - "tts": 1, - "asr": 1, + "tts": _int_env("JOB_TTS_CONCURRENCY", 2), + "asr": _int_env("JOB_ASR_CONCURRENCY", 1), } DEFAULT_QUEUE_SIZE = { @@ -51,8 +59,8 @@ DEFAULT_QUEUE_SIZE = { "compress": 8, "ocr": 8, "convert": 8, - "tts": 4, - "asr": 4, + "tts": _int_env("JOB_TTS_MAX_QUEUE", 8), + "asr": _int_env("JOB_ASR_MAX_QUEUE", 8), } @@ -94,13 +102,6 @@ def _bool_env(name: str, default: bool) -> bool: return value.strip().lower() in {"1", "true", "yes", "on"} -def _int_env(name: str, default: int) -> int: - try: - return max(1, int(os.getenv(name, str(default)))) - except (TypeError, ValueError): - return default - - def _float_env(name: str, default: float) -> float: try: return float(os.getenv(name, str(default))) @@ -248,7 +249,7 @@ class InMemoryJobManager(BaseJobManager): async with self.lock: config = _queue_config(job_type) if self.queue_counts[job_type] >= config.max_queue: - raise QueueFullError(f"{job_type} queue is full") + raise QueueFullError(job_type, config.max_queue) job_id = request_id or str(uuid.uuid4()) self.jobs[job_id] = { "job_id": job_id, @@ -261,6 +262,11 @@ class InMemoryJobManager(BaseJobManager): "cancel_requested": False, "created_at": _now_ms(), "updated_at": _now_ms(), + "started_at": 0, + "completed_at": 0, + "queue_ms": 0, + "run_ms": 0, + "total_ms": 0, } self.event_history[job_id] = [] self.queue_counts[job_type] += 1 @@ -305,6 +311,12 @@ class InMemoryJobManager(BaseJobManager): "status": job["status"], "result": job["result"], "error": job["error"], + "created_at": job.get("created_at", 0), + "started_at": job.get("started_at", 0), + "completed_at": job.get("completed_at", 0), + "queue_ms": job.get("queue_ms", 0), + "run_ms": job.get("run_ms", 0), + "total_ms": job.get("total_ms", 0), **metrics, } @@ -353,7 +365,10 @@ class InMemoryJobManager(BaseJobManager): self.queue_counts[job_type] = max(0, self.queue_counts[job_type] - 1) self.running_counts[job_type] += 1 job["status"] = "running" - job["updated_at"] = _now_ms() + started_at = _now_ms() + job["updated_at"] = started_at + job["started_at"] = started_at + job["queue_ms"] = max(0, started_at - int(job.get("created_at", started_at))) metrics = self._metrics(job_type) await self._publish(job_id, "started", {"job_id": job_id, "type": job_type, "status": "running", **metrics}) @@ -364,7 +379,14 @@ class InMemoryJobManager(BaseJobManager): def is_cancelled() -> bool: return bool(job.get("cancel_requested")) - result = await self.handlers[job_type](job["payload"], emit, is_cancelled) + job_payload = dict(job["payload"]) + job_payload["job_context"] = { + "job_id": job_id, + "created_at": int(job.get("created_at", 0) or 0), + "started_at": int(job.get("started_at", 0) or 0), + "queue_ms": int(job.get("queue_ms", 0) or 0), + } + result = await self.handlers[job_type](job_payload, emit, is_cancelled) async with self.lock: if job["cancel_requested"]: job["status"] = "cancelled" @@ -373,13 +395,35 @@ class InMemoryJobManager(BaseJobManager): return job["status"] = "completed" job["result"] = result - job["updated_at"] = _now_ms() + completed_at = _now_ms() + job["updated_at"] = completed_at + job["completed_at"] = completed_at + job["run_ms"] = max(0, completed_at - int(job.get("started_at", completed_at))) + job["total_ms"] = max(0, completed_at - int(job.get("created_at", completed_at))) metrics = self._metrics(job_type) - await self._publish(job_id, "done", {"job_id": job_id, "type": job_type, "status": "completed", "result": result, **metrics}) + await self._publish( + job_id, + "done", + { + "job_id": job_id, + "type": job_type, + "status": "completed", + "result": result, + "queue_ms": job.get("queue_ms", 0), + "run_ms": job.get("run_ms", 0), + "total_ms": job.get("total_ms", 0), + **metrics, + }, + ) except asyncio.CancelledError: async with self.lock: job["status"] = "cancelled" job["cancel_requested"] = True + completed_at = _now_ms() + job["updated_at"] = completed_at + job["completed_at"] = completed_at + job["run_ms"] = max(0, completed_at - int(job.get("started_at", completed_at))) + job["total_ms"] = max(0, completed_at - int(job.get("created_at", completed_at))) metrics = self._metrics(job_type) await self._publish(job_id, "cancelled", {"job_id": job_id, "type": job_type, "status": "cancelled", **metrics}) raise @@ -388,9 +432,26 @@ class InMemoryJobManager(BaseJobManager): async with self.lock: job["status"] = "failed" job["error"] = str(exc) - job["updated_at"] = _now_ms() + completed_at = _now_ms() + job["updated_at"] = completed_at + job["completed_at"] = completed_at + job["run_ms"] = max(0, completed_at - int(job.get("started_at", completed_at))) + job["total_ms"] = max(0, completed_at - int(job.get("created_at", completed_at))) metrics = self._metrics(job_type) - await self._publish(job_id, "error", {"job_id": job_id, "type": job_type, "status": "failed", "error": str(exc), **metrics}) + await self._publish( + job_id, + "error", + { + "job_id": job_id, + "type": job_type, + "status": "failed", + "error": str(exc), + "queue_ms": job.get("queue_ms", 0), + "run_ms": job.get("run_ms", 0), + "total_ms": job.get("total_ms", 0), + **metrics, + }, + ) finally: async with self.lock: self.running_counts[job_type] = max(0, self.running_counts[job_type] - 1) @@ -484,7 +545,7 @@ class RedisJobManager(BaseJobManager): config = _queue_config(job_type) metrics = await self._metrics(job_type) if metrics["queued_count"] >= config.max_queue: - raise QueueFullError(f"{job_type} queue is full") + raise QueueFullError(job_type, config.max_queue) job_id = request_id or str(uuid.uuid4()) created_at = _now_ms() @@ -496,6 +557,11 @@ class RedisJobManager(BaseJobManager): "error": "", "created_at": created_at, "updated_at": created_at, + "started_at": 0, + "completed_at": 0, + "queue_ms": 0, + "run_ms": 0, + "total_ms": 0, "cancel_requested": "0", } await self._set_state(job_id, state) @@ -533,6 +599,12 @@ class RedisJobManager(BaseJobManager): "error": error, "result": _json_loads(result, result), "cancel_requested": state.get("cancel_requested") == "1", + "created_at": int(state.get("created_at", "0") or 0), + "started_at": int(state.get("started_at", "0") or 0), + "completed_at": int(state.get("completed_at", "0") or 0), + "queue_ms": int(state.get("queue_ms", "0") or 0), + "run_ms": int(state.get("run_ms", "0") or 0), + "total_ms": int(state.get("total_ms", "0") or 0), **metrics, } @@ -621,6 +693,9 @@ class RedisWorker: semaphore: asyncio.Semaphore, ) -> None: job_id = fields["job_id"] + started_at = 0 + created_at = 0 + queue_ms = 0 try: state = await self.manager.get_status(job_id) if not state or state["status"] == "cancelled": @@ -629,13 +704,21 @@ class RedisWorker: await self.manager.redis.hincrby(self.manager._metrics_key(job_type), "queued_count", -1) await self.manager.redis.hincrby(self.manager._metrics_key(job_type), "running_count", 1) + started_at = _now_ms() + created_at = int(state.get("created_at", 0) or 0) + queue_ms = max(0, started_at - created_at) await self.manager._set_state(job_id, { "job_id": job_id, "request_id": state["request_id"], "type": job_type, "status": "running", - "updated_at": _now_ms(), - "created_at": state.get("created_at", _now_ms()), + "updated_at": started_at, + "created_at": created_at or started_at, + "started_at": started_at, + "completed_at": 0, + "queue_ms": queue_ms, + "run_ms": 0, + "total_ms": 0, "cancel_requested": "1" if state.get("cancel_requested") else "0", "error": "", }) @@ -643,6 +726,12 @@ class RedisWorker: await self.manager._emit_event(job_id, "started", {"job_id": job_id, "type": job_type, "status": "running", **metrics}) payload = _json_loads(fields["payload"], {}) + payload["job_context"] = { + "job_id": job_id, + "created_at": created_at, + "started_at": started_at, + "queue_ms": queue_ms, + } async def emit(event: str, data: dict[str, Any]) -> None: live_state = await self.manager.get_status(job_id) or {"status": "running"} @@ -656,6 +745,7 @@ class RedisWorker: result = await self.manager.handlers[job_type](payload, emit, is_cancelled) current = await self.manager.get_status(job_id) if current and current["status"] == "cancelled": + await self.manager.redis.xack(queue_key, group, message_id) return await self.manager._set_state(job_id, { @@ -664,16 +754,46 @@ class RedisWorker: "type": job_type, "status": "completed", "updated_at": _now_ms(), - "created_at": state.get("created_at", _now_ms()), + "created_at": created_at or started_at, + "started_at": started_at, + "completed_at": _now_ms(), + "queue_ms": queue_ms, + "run_ms": max(0, _now_ms() - started_at), + "total_ms": max(0, _now_ms() - (created_at or started_at)), "cancel_requested": "0", "error": "", "result": _json_dumps(result), }) metrics = await self.manager._metrics(job_type) - await self.manager._emit_event(job_id, "done", {"job_id": job_id, "type": job_type, "status": "completed", "result": result, **metrics}) + final_state = await self.manager.get_status(job_id) or {} + await self.manager._emit_event( + job_id, + "done", + { + "job_id": job_id, + "type": job_type, + "status": "completed", + "result": result, + "queue_ms": final_state.get("queue_ms", queue_ms), + "run_ms": final_state.get("run_ms", 0), + "total_ms": final_state.get("total_ms", 0), + **metrics, + }, + ) await self.manager.redis.xack(queue_key, group, message_id) except asyncio.CancelledError: - await self.manager.redis.hset(self.manager._state_key(job_id), mapping={"status": "cancelled", "cancel_requested": "1", "updated_at": _now_ms()}) + cancelled_at = _now_ms() + await self.manager.redis.hset( + self.manager._state_key(job_id), + mapping={ + "status": "cancelled", + "cancel_requested": "1", + "updated_at": cancelled_at, + "completed_at": cancelled_at, + "run_ms": max(0, cancelled_at - started_at), + "total_ms": max(0, cancelled_at - (created_at or started_at)), + }, + ) metrics = await self.manager._metrics(job_type) await self.manager._emit_event(job_id, "cancelled", {"job_id": job_id, "type": job_type, "status": "cancelled", **metrics}) await self.manager.redis.xack(queue_key, group, message_id) @@ -688,12 +808,31 @@ class RedisWorker: "type": job_type, "status": "failed", "updated_at": _now_ms(), - "created_at": state.get("created_at", _now_ms()) if state else _now_ms(), + "created_at": created_at or (_now_ms() if state else _now_ms()), + "started_at": started_at, + "completed_at": _now_ms(), + "queue_ms": queue_ms, + "run_ms": max(0, _now_ms() - started_at), + "total_ms": max(0, _now_ms() - (created_at or started_at)), "cancel_requested": "0", "error": str(exc), }) metrics = await self.manager._metrics(job_type) - await self.manager._emit_event(job_id, "error", {"job_id": job_id, "type": job_type, "status": "failed", "error": str(exc), **metrics}) + final_state = await self.manager.get_status(job_id) or {} + await self.manager._emit_event( + job_id, + "error", + { + "job_id": job_id, + "type": job_type, + "status": "failed", + "error": str(exc), + "queue_ms": final_state.get("queue_ms", queue_ms), + "run_ms": final_state.get("run_ms", 0), + "total_ms": final_state.get("total_ms", 0), + **metrics, + }, + ) await self.manager.redis.xack(queue_key, group, message_id) finally: self.running_tasks.pop(job_id, None) diff --git a/backend/llm.py b/backend/llm.py index 3f5f440..3da38a7 100644 --- a/backend/llm.py +++ b/backend/llm.py @@ -22,20 +22,14 @@ LLM_API_KEY = os.getenv('LLM_API_KEY', 'ollama') # Auth headers for upstream LLM service (OpenAI-compatible Bearer token) LLM_HEADERS = {'Authorization': f'Bearer {LLM_API_KEY}'} -# Model names (backward compat: fall back to OLLAMA_MODEL if LLM_MODEL not set) -_raw_model = os.getenv('LLM_MODEL') or os.getenv('OLLAMA_MODEL', 'gpt-oss:20b') -LLM_MODEL = _raw_model.strip() if _raw_model else 'gpt-oss:20b' +# Model names +DEFAULT_LLM_MODEL = 'Nex-N2-mini-mlx-OptiQ-8bit-MTP' +_raw_model = os.getenv('LLM_MODEL', DEFAULT_LLM_MODEL) +LLM_MODEL = _raw_model.strip() if _raw_model else DEFAULT_LLM_MODEL PRO_LLM_MODEL = os.getenv('PRO_LLM_MODEL', LLM_MODEL) # VLM for OCR (vision models) -VLM_MODEL = os.getenv('VLM_MODEL', 'qwen3-vl:30b') - -# Fallback for legacy OLLAMA_HOST env var (auto-convert to /v1/ path) -_legacy_host = os.getenv('OLLAMA_HOST') -if _legacy_host and not os.getenv('LLM_BASE_URL'): - base = _legacy_host.rstrip('/') - if '/v1' not in base: - LLM_BASE_URL = f"{base}/v1/" +VLM_MODEL = os.getenv('VLM_MODEL', DEFAULT_LLM_MODEL) # Normalize trailing slash for base URL LLM_BASE_URL = LLM_BASE_URL.rstrip('/') + '/' diff --git a/backend/llm_policy.py b/backend/llm_policy.py index 9fd7659..a554c4c 100644 --- a/backend/llm_policy.py +++ b/backend/llm_policy.py @@ -77,4 +77,24 @@ def resolve_llm_policy(job_type: str, request_payload: dict[str, Any], config: R temperature=0.0, thinking=None, ) + if job_type == "tts": + return LLMPolicy( + job_type=job_type, + model=config.speech_tts_model, + profile="speech_tts", + max_input_chars=config.speech_tts_max_input_chars, + max_output_tokens=0, + temperature=0.0, + thinking=None, + ) + if job_type == "asr": + return LLMPolicy( + job_type=job_type, + model=config.speech_asr_model, + profile="speech_asr", + max_input_chars=config.speech_asr_max_input_bytes, + max_output_tokens=0, + temperature=0.0, + thinking=None, + ) raise ValueError(f"unsupported llm policy job type: {job_type}") diff --git a/backend/main.py b/backend/main.py index 9720633..cfacafe 100644 --- a/backend/main.py +++ b/backend/main.py @@ -19,6 +19,7 @@ from docs_store import get_document_store from geoip import get_ip_location_text from job_handlers import ( _sanitize_converted_markdown, + _infer_convert_suffix, sanitize_inline_completion_content, ALLOWED_CONVERT_EXTENSIONS, asr_handler, @@ -296,10 +297,6 @@ def _register_handlers() -> None: manager.register_handler("tts", tts_handler) manager.register_handler("asr", asr_handler) _handlers_registered = True - - # 打印注册信息便于调试 - registered = list(getattr(manager, "handlers", {}).keys()) - logger.info("handlers registered: %s", registered) def _sse(event: str, data: dict) -> str: @@ -400,6 +397,29 @@ def _estimate_completion_chars(req: CompletionRequest | ProCompletionRequest | W return len(req.prefix or "") + len(req.suffix or "") + len(getattr(req, "instruction", "") or "") +def _estimate_job_cost(policy, raw_size: int, estimated_input_tokens: int) -> float: + if policy.profile == "speech_tts": + return round((raw_size / 1000.0) * config.speech_tts_input_cost_per_1k_chars, 8) + if policy.profile == "speech_asr": + return round((raw_size / (1024.0 * 1024.0)) * config.speech_asr_input_cost_per_mb, 8) + + pricing_in = { + "completion": config.completion_input_cost_per_1k, + "pro": config.pro_input_cost_per_1k, + "vision": config.vision_input_cost_per_1k, + }[policy.profile] + pricing_out = { + "completion": config.completion_output_cost_per_1k, + "pro": config.pro_output_cost_per_1k, + "vision": config.vision_output_cost_per_1k, + }[policy.profile] + return round( + (estimated_input_tokens / 1000.0) * pricing_in + + (policy.max_output_tokens / 1000.0) * pricing_out, + 8, + ) + + async def _prepare_llm_payload( request: Request, *, @@ -423,21 +443,7 @@ async def _prepare_llm_payload( ) raise HTTPException(status_code=400, detail=f"输入过长，超过限制 {policy.max_input_chars}") estimated_input_tokens = estimate_tokens(token_source_text if token_source_text is not None else json.dumps(request_body, ensure_ascii=False)) - pricing_in = { - "completion": config.completion_input_cost_per_1k, - "pro": config.pro_input_cost_per_1k, - "vision": config.vision_input_cost_per_1k, - }[policy.profile] - pricing_out = { - "completion": config.completion_output_cost_per_1k, - "pro": config.pro_output_cost_per_1k, - "vision": config.vision_output_cost_per_1k, - }[policy.profile] - estimated_cost = round( - (estimated_input_tokens / 1000.0) * pricing_in - + (policy.max_output_tokens / 1000.0) * pricing_out, - 8, - ) + estimated_cost = _estimate_job_cost(policy, raw_size, estimated_input_tokens) controller = get_risk_controller(config) llm_decision = await controller.check_llm(identity, scope=policy.model, estimated_cost=estimated_cost) if not llm_decision.allowed: @@ -675,7 +681,13 @@ async def convert_to_markdown(request: Request, req: ConvertRequest, auth: dict file_bytes = base64.b64decode(req.file) except Exception as exc: return JSONResponse({"error": str(exc)}, status_code=500) - input_path = persist_temp_input(file_bytes, ext or ".bin") + temp_suffix = _infer_convert_suffix(file_bytes, req.filename) + if not temp_suffix: + return JSONResponse({"error": "仅支持 txt、docx、pptx、pdf 格式"}, status_code=500) + ext = os.path.splitext(req.filename)[1].lower() + if ext != temp_suffix: + return JSONResponse({"error": "仅支持 txt、docx、pptx、pdf 格式"}, status_code=500) + input_path = persist_temp_input(file_bytes, temp_suffix) try: job_id = await _queue_job("convert", { "request_id": request_id, @@ -742,32 +754,72 @@ async def get_compress_status(task_id: str, auth: dict = Security(_authorize_req @app.post("/v1/tts-asr/tts") async def queue_tts(req: TTSJobRequest, request: Request, auth: dict = Security(_authorize_request)): del auth - request_id = _request_id(request) - job_id = await _queue_job("tts", { - "request_id": request_id, - "text": req.text, - "instruct": req.instruct, - "speaker": req.speaker, - "format": req.format, - }, request_id) + body = { + "text_chars": len((req.text or "").strip()), + "speaker": req.speaker or "Vivian", + "format": req.format or "wav", + } + try: + identity, payload = await _prepare_llm_payload( + request, + job_type="tts", + request_body=body, + raw_size=len((req.text or "").strip()), + token_source_text=req.text or "", + extra_payload={ + "text": req.text, + "instruct": req.instruct, + "speaker": req.speaker, + "format": req.format, + }, + ) + job_id = await _queue_job("tts", payload, identity.request_id) + except RiskRejected as exc: + return _risk_json_response(_request_identity(request), exc.decision) + except QueueFullError as exc: + return JSONResponse({"error": str(exc), "request_id": _request_id(request)}, status_code=429) + except JobSystemError as exc: + return JSONResponse({"error": str(exc), "request_id": _request_id(request)}, status_code=503) return await _stream_job(job_id) @app.post("/v1/tts-asr/asr") async def queue_asr(req: ASRJobRequest, request: Request, auth: dict = Security(_authorize_request)): del auth - request_id = _request_id(request) try: audio_bytes = base64.b64decode(req.audio_base64) except Exception as exc: - return JSONResponse({"error": str(exc)}, status_code=500) + return JSONResponse({"error": str(exc)}, status_code=400) input_path = persist_temp_input(audio_bytes, ".wav") try: - job_id = await _queue_job("asr", { - "request_id": request_id, - "input_path": input_path, - "language": req.language or "zh-CN", - }, request_id) + identity, payload = await _prepare_llm_payload( + request, + job_type="asr", + request_body={ + "audio_bytes": len(audio_bytes), + "language": req.language or "zh-CN", + }, + raw_size=len(audio_bytes), + token_source_text=f"audio-bytes:{len(audio_bytes)} language:{req.language or 'zh-CN'}", + extra_payload={ + "input_path": input_path, + "language": req.language or "zh-CN", + "audio_bytes": len(audio_bytes), + }, + ) + job_id = await _queue_job("asr", payload, identity.request_id) + except RiskRejected as exc: + if os.path.exists(input_path): + os.unlink(input_path) + return _risk_json_response(_request_identity(request), exc.decision) + except QueueFullError as exc: + if os.path.exists(input_path): + os.unlink(input_path) + return JSONResponse({"error": str(exc), "request_id": _request_id(request)}, status_code=429) + except JobSystemError as exc: + if os.path.exists(input_path): + os.unlink(input_path) + return JSONResponse({"error": str(exc), "request_id": _request_id(request)}, status_code=503) except Exception: if os.path.exists(input_path): os.unlink(input_path) @@ -943,20 +995,11 @@ async def download_docs_blob(request: Request, node_id: str, auth: dict = Securi return Response(content=payload.content, media_type=payload.mime_type, headers=headers) -def _register_tts_asr_routes(): - try: - from tts_asr import register_tts_asr_routes - except ModuleNotFoundError as exc: - logger.warning("Skipping TTS/ASR route registration because a dependency is missing: %s", exc) - return - except Exception as exc: - logger.warning("Skipping TTS/ASR route registration because import failed: %s", exc) - return +def _register_tts_asr_routes() -> None: + from tts_asr import LLM_BASE_URL, register_tts_asr_routes as _register_fn - try: - register_tts_asr_routes(app, include_generation_routes=False) - except Exception as exc: - logger.warning("Failed to register TTS/ASR routes: %s", exc) + logger.info("TTS/ASR routes registered with shared LLM speech backend") + _register_fn(app) _register_tts_asr_routes() @@ -964,10 +1007,13 @@ _register_tts_asr_routes() @app.on_event("shutdown") async def _shutdown_job_manager(): # pragma: no cover + from tts_asr import close_speech_client + manager = get_job_manager() close = getattr(manager, "close", None) if close is not None: await close() + await close_speech_client() if __name__ == "__main__": import uvicorn diff --git a/backend/requirements.docker.txt b/backend/requirements.docker.txt index 3bb1fd6..4d98848 100644 --- a/backend/requirements.docker.txt +++ b/backend/requirements.docker.txt @@ -8,10 +8,3 @@ python-multipart>=0.0.9 python-dotenv>=1.0.0 markitdown>=0.1.1 geoip2>=4.8.0 -numpy>=1.26.0 -torch>=2.2.0 -soundfile>=0.12.1 -scipy>=1.13.0 -qwen-tts -modelscope>=1.18.0 -faster-whisper>=1.1.0 diff --git a/backend/requirements.txt b/backend/requirements.txt index 9503b4f..58f03d3 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -6,18 +6,8 @@ redis>=5.0.0 psycopg[binary]>=3.2.0 python-multipart>=0.0.9 python-dotenv>=1.0.0 - -numpy>=1.23.0 -soundfile>=0.10.3 -torch>=1.12.0 -torchaudio>=1.12.0 -transformers>=4.25.0 -whisper>=1.0.0 -qwen-tts>=0.0.0 -modelscope>=1.20.0 - -# MLX-based ASR (Apple Silicon only) -mlx-audio>=0.4.3 +markitdown>=0.1.1 +geoip2>=4.8.0 # testing pytest>=7.0.0 diff --git a/backend/risk_config.py b/backend/risk_config.py index e6d3940..b43abda 100644 --- a/backend/risk_config.py +++ b/backend/risk_config.py @@ -75,18 +75,25 @@ class RiskConfig: pro_max_output_tokens: int pro_temperature: float web_search_model: str + speech_tts_model: str + speech_asr_model: str web_search_max_input_chars: int web_search_max_output_tokens: int web_search_temperature: float compress_max_input_chars: int compress_max_output_tokens: int ocr_max_input_bytes: int + speech_tts_max_input_chars: int + speech_asr_max_input_bytes: int completion_input_cost_per_1k: float completion_output_cost_per_1k: float pro_input_cost_per_1k: float pro_output_cost_per_1k: float vision_input_cost_per_1k: float vision_output_cost_per_1k: float + speech_tts_input_cost_per_1k_chars: float + speech_tts_output_cost_per_minute_audio: float + speech_asr_input_cost_per_mb: float def load_risk_config() -> RiskConfig: @@ -123,26 +130,33 @@ def load_risk_config() -> RiskConfig: model_circuit_breaker_failures=_int_env("RISK_MODEL_CIRCUIT_FAILURES", 8), model_circuit_ttl_seconds=_int_env("RISK_MODEL_CIRCUIT_TTL_SECONDS", 300), enforce_redis_fail_closed=_bool_env("RISK_ENFORCE_REDIS_FAIL_CLOSED", False), - completion_model=_str_env("RISK_COMPLETION_MODEL", os.getenv("LLM_MODEL", "gpt-4.1-mini")), - pro_model=_str_env("RISK_PRO_MODEL", os.getenv("PRO_LLM_MODEL", os.getenv("LLM_MODEL", "gpt-4.1"))), - vision_model=_str_env("RISK_VISION_MODEL", os.getenv("VLM_MODEL", "gpt-4.1-mini")), + completion_model=_str_env("RISK_COMPLETION_MODEL", os.getenv("LLM_MODEL", "Nex-N2-mini-mlx-OptiQ-8bit-MTP")), + pro_model=_str_env("RISK_PRO_MODEL", os.getenv("PRO_LLM_MODEL", os.getenv("LLM_MODEL", "Nex-N2-mini-mlx-OptiQ-8bit-MTP"))), + vision_model=_str_env("RISK_VISION_MODEL", os.getenv("VLM_MODEL", "Nex-N2-mini-mlx-OptiQ-8bit-MTP")), completion_max_input_chars=_int_env("RISK_COMPLETION_MAX_INPUT_CHARS", 24000), completion_max_output_tokens=_int_env("RISK_COMPLETION_MAX_OUTPUT_TOKENS", 768), completion_temperature=_float_env("RISK_COMPLETION_TEMPERATURE", 0.4), pro_max_input_chars=_int_env("RISK_PRO_MAX_INPUT_CHARS", 48000), pro_max_output_tokens=_int_env("RISK_PRO_MAX_OUTPUT_TOKENS", 2048), pro_temperature=_float_env("RISK_PRO_TEMPERATURE", 0.6), - web_search_model=_str_env("RISK_WEB_SEARCH_MODEL", os.getenv("LLM_MODEL", "gpt-4.1-mini")), + web_search_model=_str_env("RISK_WEB_SEARCH_MODEL", os.getenv("LLM_MODEL", "Nex-N2-mini-mlx-OptiQ-8bit-MTP")), + speech_tts_model=_str_env("RISK_SPEECH_TTS_MODEL", "Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit"), + speech_asr_model=_str_env("RISK_SPEECH_ASR_MODEL", "Qwen3-ASR-0.6B-8bit"), web_search_max_input_chars=_int_env("RISK_WEB_SEARCH_MAX_INPUT_CHARS", 128000), web_search_max_output_tokens=_int_env("RISK_WEB_SEARCH_MAX_OUTPUT_TOKENS", 4096), web_search_temperature=_float_env("RISK_WEB_SEARCH_TEMPERATURE", 0.4), compress_max_input_chars=_int_env("RISK_COMPRESS_MAX_INPUT_CHARS", 128000), compress_max_output_tokens=_int_env("RISK_COMPRESS_MAX_OUTPUT_TOKENS", 1536), ocr_max_input_bytes=_int_env("RISK_OCR_MAX_INPUT_BYTES", 100 * 1024 * 1024), + speech_tts_max_input_chars=_int_env("RISK_SPEECH_TTS_MAX_INPUT_CHARS", 4096), + speech_asr_max_input_bytes=_int_env("RISK_SPEECH_ASR_MAX_INPUT_BYTES", 100 * 1024 * 1024), completion_input_cost_per_1k=_float_env("RISK_COMPLETION_INPUT_COST_PER_1K", 0.0004), completion_output_cost_per_1k=_float_env("RISK_COMPLETION_OUTPUT_COST_PER_1K", 0.0016), pro_input_cost_per_1k=_float_env("RISK_PRO_INPUT_COST_PER_1K", 0.003), pro_output_cost_per_1k=_float_env("RISK_PRO_OUTPUT_COST_PER_1K", 0.012), vision_input_cost_per_1k=_float_env("RISK_VISION_INPUT_COST_PER_1K", 0.0008), vision_output_cost_per_1k=_float_env("RISK_VISION_OUTPUT_COST_PER_1K", 0.0024), + speech_tts_input_cost_per_1k_chars=_float_env("RISK_SPEECH_TTS_INPUT_COST_PER_1K_CHARS", 0.0), + speech_tts_output_cost_per_minute_audio=_float_env("RISK_SPEECH_TTS_OUTPUT_COST_PER_MINUTE_AUDIO", 0.0), + speech_asr_input_cost_per_mb=_float_env("RISK_SPEECH_ASR_INPUT_COST_PER_MB", 0.0), ) diff --git a/backend/tests/TESTING_GUIDE.md b/backend/tests/TESTING_GUIDE.md deleted file mode 100644 index 7b456f5..0000000 --- a/backend/tests/TESTING_GUIDE.md +++ /dev/null @@ -1,453 +0,0 @@ -# TTS/ASR 测试指南 - -本文档提供完整的测试脚本使用说明，包括单元测试、集成测试和macOS环境模拟测试。 - -## 测试脚本概览 - -| 脚本 | 位置 | 用途 | 需要后端服务 | -|------|------|------|--------------| -| `test_tts_asr_unit.py` | `backend/tests/` | 单元测试（设备检测、模型选择、音频处理） | 否 | -| `test_tts_asr_integration.py` | `backend/tests/` | 集成测试（API端点、完整流程） | 是 | -| `simulate_macos.py` | `backend/tests/` | macOS环境模拟（在非Mac环境测试） | 否 | - -## 快速开始 - -### 1. 单元测试（推荐首先运行） - -单元测试不需要实际运行模型或后端服务，测试代码逻辑： - -```bash -# 使用pytest运行（推荐） -pytest backend/tests/test_tts_asr_unit.py -v - -# 直接运行 -python backend/tests/test_tts_asr_unit.py - -# 运行特定测试类 -pytest backend/tests/test_tts_asr_unit.py::TestAppleSiliconDetection -v - -# 运行特定测试方法 -pytest backend/tests/test_tts_asr_unit.py::TestAppleSiliconDetection::test_is_apple_silicon_on_darwin_arm64 -v -``` - -### 2. macOS环境模拟测试 - -在非macOS环境下模拟Apple Silicon环境： - -```bash -# 运行完整模拟测试套件 -python backend/tests/simulate_macos.py --full-simulation - -# 仅模拟Apple Silicon环境并进入交互模式 -python backend/tests/simulate_macos.py --apple-silicon - -# 模拟特定设备 -python backend/tests/simulate_macos.py --device mps -python backend/tests/simulate_macos.py --device cuda - -# 运行特定测试 -python backend/tests/simulate_macos.py --test device # 设备检测 -python backend/tests/simulate_macos.py --test memory # 内存管理 -python backend/tests/simulate_macos.py --test model # 模型选择 -python backend/tests/simulate_macos.py --test audio # 音频处理 -python backend/tests/simulate_macos.py --test env # 环境变量 -``` - -### 3. 集成测试 - -集成测试需要运行后端服务： - -```bash -# 1. 启动后端服务（终端1） -python backend/main.py - -# 2. 运行集成测试（终端2） -# 运行所有测试 -python backend/tests/test_tts_asr_integration.py - -# 运行特定测试 -python backend/tests/test_tts_asr_integration.py --test config # 配置端点 -python backend/tests/test_tts_asr_integration.py --test status # 状态端点 -python backend/tests/test_tts_asr_integration.py --test warmup # 预热测试 -python backend/tests/test_tts_asr_integration.py --test tts # TTS测试 -python backend/tests/test_tts_asr_integration.py --test asr # ASR测试 -python backend/tests/test_tts_asr_integration.py --test perf # 性能测试 - -# 自定义API地址 -python backend/tests/test_tts_asr_integration.py --url http://localhost:8001 --key your-api-key -``` - -## 详细测试说明 - -### 单元测试详解 - -#### TestAppleSiliconDetection - -测试Apple Silicon检测功能： - -- `test_is_apple_silicon_on_darwin_arm64`: 在Darwin/arm64环境检测 -- `test_is_apple_silicon_on_windows`: 在Windows环境不应检测到 -- `test_is_apple_silicon_on_linux`: 在Linux环境不应检测到 - -#### TestEnvironmentVariables - -测试环境变量解析： - -- `test_default_environment_values`: 验证默认值 -- `test_custom_environment_values`: 验证自定义值 - -#### TestModelSizeSelection - -测试模型大小选择： - -- `test_whisper_model_sizes_mapping`: 模型大小映射验证 -- `test_recommended_model_size_explicit`: 显式指定大小 -- `test_invalid_model_size_falls_back`: 无效大小回退 - -#### TestAudioValidation - -测试音频验证： - -- `test_validate_empty_audio`: 空音频验证 -- `test_validate_valid_wav_header`: 有效WAV头验证 -- `test_validate_invalid_audio`: 无效音频验证 - -#### TestAudioResampling - -测试音频重采样： - -- `test_resample_same_rate`: 相同采样率 -- `test_resample_different_rate`: 不同采样率重采样 -- `test_resample_downsample`: 下采样 - -#### TestDeviceCapabilities - -测试设备能力检测： - -- `test_device_capabilities_dataclass`: 数据类验证 -- `test_device_capabilities_with_mps`: MPS设备能力 - -#### TestModelCacheCheck - -测试模型缓存检查： - -- `test_cache_check_non_offline_mode`: 非离线模式 -- `test_cache_check_offline_mode_missing`: 离线模式缺失模型 - -#### TestRequestResponseModels - -测试API模型： - -- `test_tts_request_model`: TTS请求模型 -- `test_asr_request_model`: ASR请求模型 -- `test_model_status_model`: 状态模型 - -### 集成测试详解 - -#### TTSASRIntegrationTest - -主要集成测试： - -- `test_01_config_endpoint`: 配置端点测试 -- `test_02_status_endpoint`: 状态端点测试 -- `test_03_warmup_endpoint`: 预热端点测试 -- `test_04_tts_endpoint_basic`: TTS基本功能测试 -- `test_05_asr_endpoint_basic`: ASR基本功能测试 -- `test_06_api_key_validation`: API密钥验证测试 -- `test_07_tts_long_text`: TTS长文本测试 - -#### PerformanceTest - -性能测试： - -- `test_tts_latency`: TTS延迟测试 - -### macOS模拟测试详解 - -#### MacOSSimulator类 - -提供以下模拟功能： - -- `simulate_apple_silicon()`: 模拟Darwin/arm64环境 -- `simulate_mps_device()`: 模拟MPS设备可用 -- `simulate_cuda_device()`: 模拟CUDA设备可用 -- `cleanup()`: 清理模拟环境 - -#### 独立测试函数 - -- `test_device_detection_on_apple_silicon()`: Apple Silicon设备检测 -- `test_memory_management()`: 内存管理测试 -- `test_model_size_selection()`: 模型大小选择测试 -- `test_audio_processing()`: 音频处理测试 -- `test_environment_variables()`: 环境变量测试 - -## 测试覆盖率 - -### 单元测试覆盖的功能 - -- [x] Apple Silicon检测逻辑 -- [x] 环境变量解析和默认值 -- [x] 模型大小选择和推荐 -- [x] 音频数据验证 -- [x] 音频重采样（多回退方案） -- [x] 设备能力检测数据结构 -- [x] 模型缓存检查 -- [x] API请求/响应模型 - -### 集成测试覆盖的功能 - -- [x] 配置端点（`/v1/tts-asr/config`） -- [x] 状态端点（`/v1/tts-asr/status`） -- [x] 预热端点（`/v1/tts-asr/warmup`） -- [x] TTS端点（`/v1/tts-asr/tts`） -- [x] ASR端点（`/v1/tts-asr/asr`） -- [x] API密钥验证 -- [x] 长文本处理 -- [x] 性能基准测试 - -### macOS模拟测试覆盖的场景 - -- [x] Apple Silicon环境模拟 -- [x] MPS设备模拟 -- [x] CUDA设备模拟 -- [x] 系统内存模拟 -- [x] 完整环境变量测试 - -## 常见测试场景 - -### 场景1: 开发时快速验证 - -```bash -# 快速单元测试 -pytest backend/tests/test_tts_asr_unit.py -v --tb=short - -# macOS模拟（完整） -python backend/tests/simulate_macos.py --full-simulation -``` - -### 场景2: 验证特定配置 - -```bash -# 设置环境变量后测试 -export TTS_ASR_MODEL_SIZE=small -export TTS_ASR_QUANTIZE=true - -# 运行测试 -python backend/tests/simulate_macos.py --test model -``` - -### 场景3: API功能验证 - -```bash -# 启动服务 -python backend/main.py - -# 测试配置端点 -python backend/tests/test_tts_asr_integration.py --test config - -# 测试TTS功能 -python backend/tests/test_tts_asr_integration.py --test tts - -# 测试ASR功能 -python backend/tests/test_tts_asr_integration.py --test asr -``` - -### 场景4: 性能基准测试 - -```bash -# 启动服务 -python backend/main.py - -# 运行性能测试 -python backend/tests/test_tts_asr_integration.py --test perf -``` - -## 测试输出解读 - -### 成功示例 - -``` -test_is_apple_silicon_on_darwin_arm64 ... ok -test_is_apple_silicon_on_windows ... ok -test_is_apple_silicon_on_linux ... ok - ----------------------------------------------------------------------- -Ran 3 tests in 0.005s - -OK -``` - -### 失败示例 - -``` -test_device_detection_on_apple_silicon ... FAIL - -====================================================================== -FAIL: test_device_detection_on_apple_silicon ----------------------------------------------------------------------- -Traceback (most recent call last): - File "test_tts_asr_unit.py", line 45, in test_is_apple_silicon_on_darwin_arm64 - self.assertTrue(_is_apple_silicon()) -AssertionError: False is not true - ----------------------------------------------------------------------- -Ran 1 tests in 0.002s - -FAILED (failures=1) -``` - -## 持续集成配置 - -### GitHub Actions示例 - -```yaml -name: TTS/ASR Tests - -on: [push, pull_request] - -jobs: - unit-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - name: Install dependencies - run: | - pip install -r backend/requirements.txt - pip install pytest - - name: Run unit tests - run: pytest backend/tests/test_tts_asr_unit.py -v - - name: Run macOS simulation - run: python backend/tests/simulate_macos.py --full-simulation -``` - -### pytest配置 - -创建 `pytest.ini`: - -```ini -[pytest] -testpaths = backend/tests -python_files = test_*.py -python_classes = Test* -python_functions = test_* -addopts = -v --tb=short -``` - -## 故障排查 - -### 问题1: 导入错误 - -``` -ModuleNotFoundError: No module named 'backend' -``` - -**解决方案**: - -```bash -# 确保在项目根目录运行 -cd /path/to/llm-in-text - -# 或设置PYTHONPATH -export PYTHONPATH="${PYTHONPATH}:$(pwd)" -``` - -### 问题2: 后端服务连接失败 - -``` -✗ 无法连接到服务: [Errno 111] Connection refused -``` - -**解决方案**: - -```bash -# 确保后端服务正在运行 -python backend/main.py - -# 检查端口 -lsof -i :8001 - -# 或使用自定义URL -python backend/tests/test_tts_asr_integration.py --url http://localhost:8001 -``` - -### 问题3: 模型未加载 - -``` -⚠ TTS失败（可能是模型未加载） -``` - -**解决方案**: - -这是预期行为，表示模型需要时间下载。可以： - -1. 等待模型下载完成 -2. 使用预热端点: `POST /v1/tts-asr/warmup` -3. 启用离线模式（如果模型已下载） - -### 问题4: 测试超时 - -``` -httpx.ReadTimeout: timed out -``` - -**解决方案**: - -```bash -# 增加超时时间 -export TEST_TIMEOUT=300.0 - -# 或在测试脚本中修改 -TEST_TIMEOUT = 300.0 # 5分钟 -``` - -## 最佳实践 - -1. **开发时**: 频繁运行单元测试 - ```bash - pytest backend/tests/test_tts_asr_unit.py -v --tb=short - ``` - -2. **提交前**: 运行完整测试套件 - ```bash - pytest backend/tests/test_tts_asr_unit.py -v - python backend/tests/simulate_macos.py --full-simulation - ``` - -3. **部署前**: 运行集成测试 - ```bash - python backend/tests/test_tts_asr_integration.py - ``` - -4. **调试时**: 使用详细输出 - ```bash - pytest backend/tests/test_tts_asr_unit.py -v -s --tb=long - ``` - -## 测试报告 - -生成测试覆盖率报告： - -```bash -# 安装coverage -pip install pytest-cov - -# 运行并生成报告 -pytest backend/tests/test_tts_asr_unit.py --cov=backend.tts_asr --cov-report=html - -# 查看报告 -open htmlcov/index.html -``` - -## 相关文档 - -- [TTS/ASR修复说明](./TTS_ASR_MACOS_FIX.md) -- [环境变量配置](../README.md#ttsasr环境变量配置) -- [API文档](../README.md#api接口) - ---- - -**更新日期**: 2026-04-06 -**维护者**: 项目开发团队 diff --git a/backend/tests/benchmark_tts_asr.py b/backend/tests/benchmark_tts_asr.py new file mode 100644 index 0000000..30ace0c --- /dev/null +++ b/backend/tests/benchmark_tts_asr.py @@ -0,0 +1,209 @@ +"""Lightweight benchmark for TTS/ASR queueing and API throughput. + +This benchmark uses the FastAPI app with a mocked upstream speech API so it +measures this project's queueing, request handling, and SSE delivery cost +without requiring a real external model endpoint. +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import json +import os +import statistics +import time +from pathlib import Path + +import httpx + +BACKEND_DIR = Path(__file__).resolve().parents[1] +if str(BACKEND_DIR) not in __import__("sys").path: + __import__("sys").path.insert(0, str(BACKEND_DIR)) + +import main # noqa: E402 +import tts_asr # noqa: E402 +from job_system import reset_job_manager # noqa: E402 + + +def _wav_bytes(duration_ms: int = 320) -> bytes: + sample_rate = 16000 + frames = max(1, int(sample_rate * duration_ms / 1000)) + data = b"".join((i % 32768).to_bytes(2, "little", signed=False) for i in range(frames)) + data_size = len(data) + return ( + b"RIFF" + (36 + data_size).to_bytes(4, "little") + + b"WAVE" + + b"fmt " + (16).to_bytes(4, "little") + + (1).to_bytes(2, "little") + + (1).to_bytes(2, "little") + + sample_rate.to_bytes(4, "little") + + sample_rate.to_bytes(4, "little") + + (2).to_bytes(2, "little") + + (16).to_bytes(2, "little") + + b"data" + data_size.to_bytes(4, "little") + + data + ) + + +def _parse_sse_done(text: str) -> dict: + for chunk in reversed([item for item in text.split("\n\n") if item.strip()]): + event = "" + data = "" + for line in chunk.splitlines(): + if line.startswith("event:"): + event = line.split(":", 1)[1].strip() + elif line.startswith("data:"): + data = line.split(":", 1)[1].strip() + if event == "done" and data: + payload = json.loads(data) + result = dict(payload.get("result") or {}) + for key in ("queue_ms", "run_ms", "total_ms", "queued_count", "running_count", "busy_level", "busy_ratio"): + if key in payload: + result[key] = payload[key] + return result + raise RuntimeError("done event not found") + + +def _percentile(values: list[float], q: float) -> float: + if not values: + return 0.0 + if len(values) == 1: + return values[0] + index = (len(values) - 1) * q + lower = int(index) + upper = min(lower + 1, len(values) - 1) + if lower == upper: + return values[lower] + weight = index - lower + return values[lower] * (1 - weight) + values[upper] * weight + + +async def _build_mock_client(tts_delay_ms: int, asr_delay_ms: int) -> httpx.AsyncClient: + async def transport(request: httpx.Request): + if request.url.path.endswith("/audio/speech"): + await asyncio.sleep(tts_delay_ms / 1000.0) + return httpx.Response(200, content=_wav_bytes(420), headers={"x-request-id": "bench-tts"}, request=request) + await asyncio.sleep(asr_delay_ms / 1000.0) + return httpx.Response(200, json={"text": "benchmark transcript", "language": "zh"}, headers={"x-request-id": "bench-asr"}, request=request) + + return httpx.AsyncClient( + base_url="https://benchmark.example/v1/", + transport=httpx.MockTransport(transport), + ) + + +async def _run_case(case_name: str, concurrency: int, request_count: int, audio_b64: str | None = None) -> dict: + results: list[dict] = [] + latencies: list[float] = [] + + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=main.app), + base_url="http://testserver", + timeout=120.0, + ) as client: + semaphore = asyncio.Semaphore(concurrency) + + async def fire(index: int) -> None: + async with semaphore: + started = time.perf_counter() + if case_name == "tts": + response = await client.post( + "/v1/tts-asr/tts", + json={"text": f"第 {index} 条基准文本", "speaker": "Vivian", "format": "wav"}, + ) + else: + response = await client.post( + "/v1/tts-asr/asr", + json={"audio_base64": audio_b64, "language": "zh-CN"}, + ) + response.raise_for_status() + payload = _parse_sse_done(response.text) + latencies.append((time.perf_counter() - started) * 1000.0) + results.append(payload) + + await asyncio.gather(*(fire(index) for index in range(request_count))) + + queue_values = sorted(float(item.get("queue_ms", 0) or 0) for item in results) + run_values = sorted(float(item.get("run_ms", 0) or 0) for item in results) + total_values = sorted(float(item.get("total_ms", 0) or 0) for item in results) + latency_values = sorted(latencies) + elapsed_sum_ms = sum(latency_values) + return { + "case": case_name, + "requests": request_count, + "concurrency": concurrency, + "avg_latency_ms": round(statistics.fmean(latency_values), 2), + "p95_latency_ms": round(_percentile(latency_values, 0.95), 2), + "avg_queue_ms": round(statistics.fmean(queue_values), 2), + "p95_queue_ms": round(_percentile(queue_values, 0.95), 2), + "avg_run_ms": round(statistics.fmean(run_values), 2), + "p95_run_ms": round(_percentile(run_values, 0.95), 2), + "avg_total_ms": round(statistics.fmean(total_values), 2), + "p95_total_ms": round(_percentile(total_values, 0.95), 2), + "throughput_rps_estimate": round((request_count * 1000.0) / max(latency_values[-1], elapsed_sum_ms / max(request_count, 1)), 2), + } + + +async def main_async(args) -> None: + os.environ["JOB_BACKEND"] = "memory" + os.environ["JOB_TTS_CONCURRENCY"] = str(args.tts_workers) + os.environ["JOB_TTS_MAX_QUEUE"] = str(max(args.tts_requests, args.tts_workers)) + os.environ["JOB_ASR_CONCURRENCY"] = str(args.asr_workers) + os.environ["JOB_ASR_MAX_QUEUE"] = str(max(args.asr_requests, args.asr_workers)) + reset_job_manager() + + mock_client = await _build_mock_client(args.tts_delay_ms, args.asr_delay_ms) + tts_asr._httpx_client = mock_client + try: + audio_b64 = base64.b64encode(_wav_bytes(args.audio_duration_ms)).decode("utf-8") + tts_stats = await _run_case("tts", args.tts_concurrency, args.tts_requests) + asr_stats = await _run_case("asr", args.asr_concurrency, args.asr_requests, audio_b64=audio_b64) + finally: + await mock_client.aclose() + tts_asr._httpx_client = None + reset_job_manager() + + print( + json.dumps( + { + "benchmark_date": time.strftime("%Y-%m-%d %H:%M:%S"), + "assumptions": { + "upstream_tts_delay_ms": args.tts_delay_ms, + "upstream_asr_delay_ms": args.asr_delay_ms, + "job_backend": "memory", + }, + "tts": tts_stats, + "asr": asr_stats, + "recommended_defaults": { + "JOB_TTS_CONCURRENCY": args.tts_workers, + "JOB_TTS_MAX_QUEUE": max(16, args.tts_workers * 4), + "JOB_ASR_CONCURRENCY": args.asr_workers, + "JOB_ASR_MAX_QUEUE": max(8, args.asr_workers * 4), + "TTS_ASR_MAX_CONNECTIONS": max(24, (args.tts_workers + args.asr_workers) * 4), + "TTS_ASR_MAX_KEEPALIVE_CONNECTIONS": max(12, (args.tts_workers + args.asr_workers) * 2), + }, + }, + ensure_ascii=False, + indent=2, + ) + ) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--tts-delay-ms", type=int, default=120) + parser.add_argument("--asr-delay-ms", type=int, default=280) + parser.add_argument("--tts-workers", type=int, default=4) + parser.add_argument("--asr-workers", type=int, default=2) + parser.add_argument("--tts-concurrency", type=int, default=8) + parser.add_argument("--asr-concurrency", type=int, default=4) + parser.add_argument("--tts-requests", type=int, default=32) + parser.add_argument("--asr-requests", type=int, default=16) + parser.add_argument("--audio-duration-ms", type=int, default=320) + return parser.parse_args() + + +if __name__ == "__main__": + asyncio.run(main_async(parse_args())) diff --git a/backend/tests/quick_verify.py b/backend/tests/quick_verify.py deleted file mode 100644 index e6a8785..0000000 --- a/backend/tests/quick_verify.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -快速验证脚本 -验证TTS/ASR模块修复是否正确应用 - -运行方式: - python backend/tests/quick_verify.py -""" - -import os -import sys -from pathlib import Path - -# 设置控制台编码 -if sys.platform == 'win32': - import io - sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') - sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8') - -# 确保可以导入backend模块 -script_path = Path(__file__).resolve() -project_root = script_path.parent.parent.parent -sys.path.insert(0, str(project_root)) - -print(f"项目根目录: {project_root}") -print(f"脚本路径: {script_path}") - - -def check_file_exists(filepath: str, description: str) -> bool: - """检查文件是否存在""" - full_path = project_root / filepath - exists = full_path.exists() - status = "[OK]" if exists else "[FAIL]" - print(f"{status} {description}: {filepath} (完整路径: {full_path})") - return exists - - -def check_function_exists(module_name: str, function_name: str) -> bool: - """检查函数是否存在""" - try: - module = __import__(module_name, fromlist=[function_name]) - exists = hasattr(module, function_name) - status = "[OK]" if exists else "[FAIL]" - print(f"{status} 函数存在: {module_name}.{function_name}") - return exists - except Exception as e: - print(f"[FAIL] 导入失败: {module_name} - {e}") - return False - - -def check_environment_variable(var_name: str, expected_default: str) -> bool: - """检查环境变量默认值""" - try: - # 清除可能存在的环境变量 - original_value = os.environ.get(var_name) - if var_name in os.environ: - del os.environ[var_name] - - # 重新导入模块 - if 'backend.tts_asr' in sys.modules: - del sys.modules['backend.tts_asr'] - - from backend.tts_asr import ( - TTS_ASR_DEVICE, TTS_ASR_MODEL_SIZE, TTS_ASR_QUANTIZE, - TTS_ASR_OFFLINE_MODE, TTS_ASR_WARMUP, TTS_ASR_WARMUP_TIMEOUT, - TTS_ASR_IDLE_TIMEOUT, TTS_ASR_MPS_MEMORY_LIMIT_MB - ) - - var_map = { - 'TTS_ASR_DEVICE': TTS_ASR_DEVICE, - 'TTS_ASR_MODEL_SIZE': TTS_ASR_MODEL_SIZE, - 'TTS_ASR_QUANTIZE': TTS_ASR_QUANTIZE, - 'TTS_ASR_OFFLINE_MODE': TTS_ASR_OFFLINE_MODE, - 'TTS_ASR_WARMUP': TTS_ASR_WARMUP, - 'TTS_ASR_WARMUP_TIMEOUT': TTS_ASR_WARMUP_TIMEOUT, - 'TTS_ASR_IDLE_TIMEOUT': TTS_ASR_IDLE_TIMEOUT, - 'TTS_ASR_MPS_MEMORY_LIMIT_MB': TTS_ASR_MPS_MEMORY_LIMIT_MB, - } - - actual_value = var_map.get(var_name) - if var_name == 'TTS_ASR_MODEL_SIZE': - expected = 'auto' - elif var_name == 'TTS_ASR_QUANTIZE': - expected = False - elif var_name == 'TTS_ASR_OFFLINE_MODE': - expected = False - elif var_name == 'TTS_ASR_WARMUP': - expected = True - elif var_name == 'TTS_ASR_WARMUP_TIMEOUT': - expected = 120 - elif var_name == 'TTS_ASR_IDLE_TIMEOUT': - expected = 0 - elif var_name == 'TTS_ASR_MPS_MEMORY_LIMIT_MB': - expected = 8192 - else: - expected = expected_default - - matches = actual_value == expected - status = "[OK]" if matches else "[FAIL]" - print(f"{status} 环境变量默认值: {var_name} = {actual_value} (预期: {expected})") - return matches - - except Exception as e: - print(f"[FAIL] 检查环境变量失败: {var_name} - {e}") - return False - - -def main(): - print("="*70) - print("TTS/ASR模块快速验证") - print("="*70) - - checks = [] - - # 1. 检查文件 - print("\n[1] 文件检查") - print("-"*70) - checks.append(check_file_exists("backend/tts_asr.py", "主模块文件")) - checks.append(check_file_exists("backend/tests/test_tts_asr_unit.py", "单元测试")) - checks.append(check_file_exists("backend/tests/test_tts_asr_integration.py", "集成测试")) - checks.append(check_file_exists("backend/tests/simulate_macos.py", "macOS模拟工具")) - checks.append(check_file_exists("backend/tests/TESTING_GUIDE.md", "测试指南")) - checks.append(check_file_exists("backend/TTS_ASR_MACOS_FIX.md", "修复文档")) - - # 2. 检查核心函数 - print("\n[2] 核心函数检查") - print("-"*70) - checks.append(check_function_exists("backend.tts_asr", "_is_apple_silicon")) - checks.append(check_function_exists("backend.tts_asr", "_detect_device_capabilities")) - checks.append(check_function_exists("backend.tts_asr", "_get_recommended_model_size")) - checks.append(check_function_exists("backend.tts_asr", "_validate_audio_data")) - checks.append(check_function_exists("backend.tts_asr", "_resample_audio_robust")) - checks.append(check_function_exists("backend.tts_asr", "_check_model_cached")) - - # 3. 检查数据类 - print("\n[3] 数据类检查") - print("-"*70) - checks.append(check_function_exists("backend.tts_asr", "DeviceCapabilities")) - checks.append(check_function_exists("backend.tts_asr", "ModelStatus")) - - # 4. 检查环境变量 - print("\n[4] 环境变量默认值检查") - print("-"*70) - checks.append(check_environment_variable("TTS_ASR_DEVICE", "auto")) - checks.append(check_environment_variable("TTS_ASR_MODEL_SIZE", "auto")) - checks.append(check_environment_variable("TTS_ASR_QUANTIZE", "false")) - checks.append(check_environment_variable("TTS_ASR_OFFLINE_MODE", "false")) - - # 5. 检查常量 - print("\n[5] 常量检查") - print("-"*70) - try: - from backend.tts_asr import WHISPER_MODEL_SIZES, APPLE_SILICON_DEFAULT_SIZE - expected_sizes = ['tiny', 'base', 'small', 'medium', 'large', 'turbo'] - sizes_match = list(WHISPER_MODEL_SIZES.keys()) == expected_sizes - status = "[OK]" if sizes_match else "[FAIL]" - print(f"{status} WHISPER_MODEL_SIZES: {list(WHISPER_MODEL_SIZES.keys())}") - checks.append(sizes_match) - - size_match = APPLE_SILICON_DEFAULT_SIZE == 'small' - status = "[OK]" if size_match else "[FAIL]" - print(f"{status} APPLE_SILICON_DEFAULT_SIZE: {APPLE_SILICON_DEFAULT_SIZE}") - checks.append(size_match) - except Exception as e: - print(f"[FAIL] 常量检查失败: {e}") - checks.extend([False, False]) - - # 汇总结果 - print("\n" + "="*70) - print("验证结果") - print("="*70) - - total = len(checks) - passed = sum(checks) - - print(f"通过: {passed}/{total}") - - if all(checks): - print("\n[SUCCESS] 所有验证通过！TTS/ASR模块修复已正确应用。") - return 0 - else: - print("\n[FAILED] 部分验证失败，请检查上述错误。") - return 1 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/backend/tests/run_tests.py b/backend/tests/run_tests.py index c2aab45..605c677 100644 --- a/backend/tests/run_tests.py +++ b/backend/tests/run_tests.py @@ -1,16 +1,7 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -TTS/ASR测试运行器 -便捷地运行各种测试组合 +"""Speech test runner for the current API-based TTS/ASR stack.""" -运行方式: - python backend/tests/run_tests.py --help - python backend/tests/run_tests.py unit - python backend/tests/run_tests.py integration - python backend/tests/run_tests.py simulate - python backend/tests/run_tests.py all -""" +from __future__ import annotations import argparse import os @@ -19,186 +10,72 @@ import sys from pathlib import Path -def run_command(cmd: list, cwd: str = None) -> int: - """运行命令并返回退出码""" +def run_command(cmd: list[str], cwd: str | None = None) -> int: print(f"\n执行: {' '.join(cmd)}") print("-" * 70) - result = subprocess.run(cmd, cwd=cwd) - return result.returncode + return subprocess.run(cmd, cwd=cwd).returncode def run_unit_tests(verbose: bool = False) -> int: - """运行单元测试""" - print("\n" + "="*70) - print("运行单元测试") - print("="*70) - - cmd = ['pytest', 'backend/tests/test_tts_asr_unit.py'] + cmd = ["pytest", "backend/tests/test_tts_asr.py"] if verbose: - cmd.append('-v') - + cmd.append("-v") return run_command(cmd) -def run_integration_tests(test_type: str = None, url: str = None, key: str = None) -> int: - """运行集成测试""" - print("\n" + "="*70) - print("运行集成测试") - print("="*70) - - cmd = ['python', 'backend/tests/test_tts_asr_integration.py'] - - if test_type: - cmd.extend(['--test', test_type]) - - if url: - cmd.extend(['--url', url]) - - if key: - cmd.extend(['--key', key]) - +def run_benchmark(extra_args: list[str] | None = None) -> int: + cmd = ["python", "backend/tests/benchmark_tts_asr.py"] + if extra_args: + cmd.extend(extra_args) return run_command(cmd) -def run_simulation(test_type: str = None) -> int: - """运行macOS模拟测试""" - print("\n" + "="*70) - print("运行macOS环境模拟测试") - print("="*70) - - if test_type == 'full': - cmd = ['python', 'backend/tests/simulate_macos.py', '--full-simulation'] - elif test_type: - cmd = ['python', 'backend/tests/simulate_macos.py', '--test', test_type] - else: - cmd = ['python', 'backend/tests/simulate_macos.py', '--full-simulation'] - - return run_command(cmd) +def run_all(verbose: bool = False) -> int: + results = [ + ("单元测试", run_unit_tests(verbose=verbose)), + ("基准测试", run_benchmark()), + ] - -def run_all_tests(url: str = None, key: str = None) -> int: - """运行所有测试""" - print("\n" + "="*70) - print("运行完整测试套件") - print("="*70) - - results = [] - - # 1. 单元测试 - print("\n[1/3] 单元测试") - results.append(("单元测试", run_unit_tests(verbose=True))) - - # 2. macOS模拟测试 - print("\n[2/3] macOS模拟测试") - results.append(("macOS模拟", run_simulation(test_type='full'))) - - # 3. 集成测试（如果服务可用） - print("\n[3/3] 集成测试") - print("注意: 集成测试需要后端服务运行中") - response = input("是否继续运行集成测试? [y/N]: ") - - if response.lower() == 'y': - results.append(("集成测试", run_integration_tests(url=url, key=key))) - else: - print("跳过集成测试") - results.append(("集成测试", 0)) - - # 汇总结果 - print("\n" + "="*70) + print("\n" + "=" * 70) print("测试结果汇总") - print("="*70) - - total_passed = 0 + print("=" * 70) + passed = 0 for name, code in results: - status = "✓ 通过" if code == 0 else "✗ 失败" - print(f"{name}: {status}") - if code == 0: - total_passed += 1 - - print("\n" + "-"*70) - print(f"总计: {total_passed}/{len(results)} 测试套件通过") - print("="*70) - - return 0 if all(code == 0 for _, code in results) else 1 + ok = code == 0 + passed += int(ok) + print(f"{name}: {'✓ 通过' if ok else '✗ 失败'}") + print("-" * 70) + print(f"总计: {passed}/{len(results)} 通过") + return 0 if passed == len(results) else 1 -def main(): - parser = argparse.ArgumentParser( - description='TTS/ASR测试运行器', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -示例: - # 运行单元测试 - python backend/tests/run_tests.py unit - - # 运行集成测试 - python backend/tests/run_tests.py integration - - # 运行macOS模拟测试 - python backend/tests/run_tests.py simulate - - # 运行所有测试 - python backend/tests/run_tests.py all - - # 运行特定集成测试 - python backend/tests/run_tests.py integration --test config - - # 运行特定模拟测试 - python backend/tests/run_tests.py simulate --test device - """ - ) - - subparsers = parser.add_subparsers(dest='command', help='测试类型') - - # 单元测试 - unit_parser = subparsers.add_parser('unit', help='运行单元测试') - unit_parser.add_argument('-v', '--verbose', action='store_true', help='详细输出') - - # 集成测试 - integration_parser = subparsers.add_parser('integration', help='运行集成测试') - integration_parser.add_argument('--test', choices=[ - 'config', 'status', 'warmup', 'tts', 'asr', 'perf' - ], help='运行特定测试') - integration_parser.add_argument('--url', default='http://localhost:8001', help='API URL') - integration_parser.add_argument('--key', default='your-secret-key-here', help='API密钥') - - # macOS模拟测试 - simulate_parser = subparsers.add_parser('simulate', help='运行macOS模拟测试') - simulate_parser.add_argument('--test', choices=[ - 'device', 'memory', 'model', 'audio', 'env', 'full' - ], help='运行特定测试') - - # 所有测试 - all_parser = subparsers.add_parser('all', help='运行所有测试') - all_parser.add_argument('--url', default='http://localhost:8001', help='API URL') - all_parser.add_argument('--key', default='your-secret-key-here', help='API密钥') - +def main() -> int: + parser = argparse.ArgumentParser(description="当前 API 化 TTS/ASR 测试运行器") + subparsers = parser.add_subparsers(dest="command", help="测试类型") + + unit_parser = subparsers.add_parser("unit", help="运行当前 TTS/ASR 单元测试") + unit_parser.add_argument("-v", "--verbose", action="store_true", help="详细输出") + + benchmark_parser = subparsers.add_parser("benchmark", help="运行当前 TTS/ASR benchmark") + benchmark_parser.add_argument("benchmark_args", nargs="*", help="透传给 benchmark_tts_asr.py") + + all_parser = subparsers.add_parser("all", help="运行当前 TTS/ASR 单元测试和 benchmark") + all_parser.add_argument("-v", "--verbose", action="store_true", help="详细输出") + args = parser.parse_args() - - # 确保在项目根目录 project_root = Path(__file__).parent.parent.parent os.chdir(project_root) - - if args.command == 'unit': + + if args.command == "unit": return run_unit_tests(verbose=args.verbose) - - elif args.command == 'integration': - return run_integration_tests( - test_type=args.test, - url=args.url, - key=args.key - ) - - elif args.command == 'simulate': - return run_simulation(test_type=args.test) - - elif args.command == 'all': - return run_all_tests(url=args.url, key=args.key) - - else: - parser.print_help() - return 0 + if args.command == "benchmark": + return run_benchmark(extra_args=args.benchmark_args) + if args.command == "all": + return run_all(verbose=args.verbose) + + parser.print_help() + return 0 -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/backend/tests/simulate_macos.py b/backend/tests/simulate_macos.py deleted file mode 100644 index 5f2bb67..0000000 --- a/backend/tests/simulate_macos.py +++ /dev/null @@ -1,504 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -macOS环境模拟测试工具 -在非macOS环境下模拟Apple Silicon环境进行测试 - -运行方式: - python backend/tests/simulate_macos.py --help - python backend/tests/simulate_macos.py --device mps - python backend/tests/simulate_macos.py --apple-silicon - python backend/tests/simulate_macos.py --full-simulation -""" - -import argparse -import os -import platform -import sys -from unittest.mock import patch -import numpy as np - - -class MacOSSimulator: - """macOS环境模拟器""" - - def __init__(self): - self.original_platform_system = platform.system - self.original_platform_machine = platform.machine - self.patches = [] - - def simulate_apple_silicon(self): - """模拟Apple Silicon环境""" - print("\n" + "="*70) - print("模拟 Apple Silicon 环境") - print("="*70) - - # 模拟Darwin系统和arm64架构 - self.patches.append(patch('platform.system', return_value='Darwin')) - self.patches.append(patch('platform.machine', return_value='arm64')) - - for p in self.patches: - p.start() - - print("✓ 平台: Darwin (macOS)") - print("✓ 架构: arm64 (Apple Silicon)") - - def simulate_mps_device(self): - """模拟MPS设备可用""" - print("\n" + "="*70) - print("模拟 MPS 设备") - print("="*70) - - # 创建模拟的torch.backends.mps - mock_mps = type('MockMPS', (), { - 'is_available': lambda: True, - 'is_built': lambda: True, - 'empty_cache': lambda: None - })() - - mock_backends = type('MockBackends', (), { - 'mps': mock_mps - })() - - # 模拟torch模块 - mock_torch = type('MockTorch', (), { - 'backends': mock_backends, - 'mps': mock_mps, - 'randn': lambda *args, **kwargs: np.random.randn(*args), - 'mm': lambda a, b: np.dot(a, b), - 'empty_cache': lambda: None - })() - - self.patches.append(patch('torch', mock_torch)) - self.patches.append(patch('torch.backends.mps.is_available', return_value=True)) - self.patches.append(patch('torch.backends.mps.is_built', return_value=True)) - - for p in self.patches[-3:]: - p.start() - - print("✓ MPS 可用: True") - print("✓ MPS 已编译: True") - - def simulate_cuda_device(self): - """模拟CUDA设备可用""" - print("\n" + "="*70) - print("模拟 CUDA 设备") - print("="*70) - - mock_cuda = type('MockCUDA', (), { - 'is_available': lambda: True, - 'device_count': lambda: 1, - 'get_device_properties': lambda n: type('Props', (), {'total_memory': 8*1024*1024*1024})(), - 'empty_cache': lambda: None - })() - - self.patches.append(patch('torch.cuda', mock_cuda)) - self.patches.append(patch('torch.cuda.is_available', return_value=True)) - - for p in self.patches[-2:]: - p.start() - - print("✓ CUDA 可用: True") - print("✓ GPU 数量: 1") - print("✓ 显存: 8 GB") - - def cleanup(self): - """清理所有补丁""" - for p in self.patches: - p.stop() - self.patches.clear() - print("\n✓ 已清理模拟环境") - - -def test_device_detection_on_apple_silicon(): - """测试Apple Silicon设备检测""" - print("\n测试1: Apple Silicon 设备检测") - print("-"*70) - - simulator = MacOSSimulator() - try: - simulator.simulate_apple_silicon() - simulator.simulate_mps_device() - - # 设置环境变量 - os.environ['TTS_ASR_DEVICE'] = 'auto' - os.environ['TTS_ASR_MODEL_SIZE'] = 'auto' - - # 重新导入模块以应用模拟 - if 'backend.tts_asr' in sys.modules: - del sys.modules['backend.tts_asr'] - - from backend.tts_asr import ( - _is_apple_silicon, - _detect_device_capabilities, - _get_recommended_model_size - ) - - # 测试Apple Silicon检测 - assert _is_apple_silicon(), "应该检测到Apple Silicon" - print("✓ Apple Silicon 检测: 通过") - - # 测试设备能力检测 - caps = _detect_device_capabilities() - print(f"✓ 设备: {caps.device}") - print(f"✓ MPS 可用: {caps.mps_available}") - print(f"✓ 推荐模型大小: {caps.recommended_model_size}") - - # 测试模型大小推荐 - recommended_size = _get_recommended_model_size() - assert recommended_size in ['small', 'tiny', 'base'], \ - f"Apple Silicon应推荐小模型，但推荐了 {recommended_size}" - print(f"✓ 推荐模型大小: {recommended_size}") - - print("\n✓ 测试通过") - return True - - except Exception as e: - print(f"\n✗ 测试失败: {e}") - import traceback - traceback.print_exc() - return False - finally: - simulator.cleanup() - - -def test_memory_management(): - """测试内存管理""" - print("\n测试2: 内存管理") - print("-"*70) - - simulator = MacOSSimulator() - try: - simulator.simulate_apple_silicon() - simulator.simulate_mps_device() - - # 模拟系统内存 - import psutil - original_virtual_memory = psutil.virtual_memory - - def mock_virtual_memory(): - mock_mem = type('MockMemory', (), { - 'total': 16 * 1024 * 1024 * 1024 # 16GB - })() - return mock_mem - - self.patches.append(patch('psutil.virtual_memory', mock_virtual_memory)) - - from backend.tts_asr import _get_system_memory_mb, TTS_ASR_MPS_MEMORY_LIMIT_MB - - mem_mb = _get_system_memory_mb() - print(f"✓ 系统内存: {mem_mb} MB") - - # 计算预期的MPS内存限制（60%） - expected_limit = int(mem_mb * 0.6) - print(f"✓ 预期MPS限制: {expected_limit} MB (60%)") - print(f"✓ 配置MPS限制: {TTS_ASR_MPS_MEMORY_LIMIT_MB} MB") - - print("\n✓ 测试通过") - return True - - except Exception as e: - print(f"\n✗ 测试失败: {e}") - import traceback - traceback.print_exc() - return False - finally: - simulator.cleanup() - - -def test_model_size_selection(): - """测试模型大小选择""" - print("\n测试3: 模型大小选择") - print("-"*70) - - test_cases = [ - ('auto', 'Apple Silicon默认'), - ('tiny', '最小模型'), - ('small', '推荐模型'), - ('medium', '中等模型'), - ('large', '大模型'), - ('turbo', 'turbo模型'), - ] - - from backend.tts_asr import WHISPER_MODEL_SIZES, _get_recommended_model_size - - for size, desc in test_cases: - os.environ['TTS_ASR_MODEL_SIZE'] = size - - # 重新加载模块 - if 'backend.tts_asr' in sys.modules: - del sys.modules['backend.tts_asr'] - - from backend.tts_asr import _get_recommended_model_size - - if size == 'auto': - # 自动选择 - recommended = _get_recommended_model_size() - print(f"✓ {desc}: {recommended}") - else: - # 显式选择 - os.environ['TTS_ASR_MODEL_SIZE'] = size - result = _get_recommended_model_size() - assert result == size, f"应该返回 {size}，但返回了 {result}" - print(f"✓ {desc}: {size} -> {WHISPER_MODEL_SIZES[size]}") - - print("\n✓ 测试通过") - return True - - -def test_audio_processing(): - """测试音频处理""" - print("\n测试4: 音频处理") - print("-"*70) - - from backend.tts_asr import ( - _validate_audio_data, - _resample_audio_robust - ) - - # 测试音频验证 - test_cases = [ - (b'', False, "空数据"), - (b'short', False, "太短"), - (b'RIFF' + b'\x00' * 40, True, "有效WAV头"), - ] - - for data, expected, desc in test_cases: - result = _validate_audio_data(data) - assert result == expected, f"{desc}: 预期 {expected}，得到 {result}" - print(f"✓ 音频验证 ({desc}): {'通过' if result == expected else '失败'}") - - # 测试重采样 - audio_16k = np.sin(np.linspace(0, 2*np.pi, 16000)).astype(np.float32) - - # 16k -> 48k - audio_48k = _resample_audio_robust(audio_16k, 16000, 48000) - assert len(audio_48k) == 48000, f"48kHz音频长度错误: {len(audio_48k)}" - print(f"✓ 重采样 (16k -> 48k): 长度 {len(audio_16k)} -> {len(audio_48k)}") - - # 48k -> 16k - audio_back = _resample_audio_robust(audio_48k, 48000, 16000) - assert len(audio_back) == 16000, f"16kHz音频长度错误: {len(audio_back)}" - print(f"✓ 重采样 (48k -> 16k): 长度 {len(audio_48k)} -> {len(audio_back)}") - - print("\n✓ 测试通过") - return True - - -def test_environment_variables(): - """测试环境变量""" - print("\n测试5: 环境变量配置") - print("-"*70) - - # 清理环境变量 - env_vars = [ - 'TTS_ASR_DEVICE', 'TTS_ASR_MODEL_SIZE', 'TTS_ASR_QUANTIZE', - 'TTS_ASR_OFFLINE_MODE', 'TTS_ASR_WARMUP', 'TTS_ASR_WARMUP_TIMEOUT', - 'TTS_ASR_IDLE_TIMEOUT', 'TTS_ASR_MPS_MEMORY_LIMIT_MB' - ] - - original_values = {} - for var in env_vars: - original_values[var] = os.environ.get(var) - if var in os.environ: - del os.environ[var] - - try: - # 测试默认值 - from backend.tts_asr import ( - TTS_ASR_DEVICE, TTS_ASR_MODEL_SIZE, TTS_ASR_QUANTIZE, - TTS_ASR_OFFLINE_MODE, TTS_ASR_WARMUP, TTS_ASR_WARMUP_TIMEOUT, - TTS_ASR_IDLE_TIMEOUT, TTS_ASR_MPS_MEMORY_LIMIT_MB - ) - - defaults = { - 'TTS_ASR_DEVICE': 'auto', - 'TTS_ASR_MODEL_SIZE': 'auto', - 'TTS_ASR_QUANTIZE': False, - 'TTS_ASR_OFFLINE_MODE': False, - 'TTS_ASR_WARMUP': True, - 'TTS_ASR_WARMUP_TIMEOUT': 120, - 'TTS_ASR_IDLE_TIMEOUT': 0, - 'TTS_ASR_MPS_MEMORY_LIMIT_MB': 8192, - } - - for var, expected in defaults.items(): - actual = locals()[var] - assert actual == expected, f"{var}: 预期 {expected}，得到 {actual}" - print(f"✓ {var} = {actual}") - - # 测试自定义值 - print("\n自定义配置测试:") - os.environ['TTS_ASR_MODEL_SIZE'] = 'small' - os.environ['TTS_ASR_QUANTIZE'] = 'true' - os.environ['TTS_ASR_OFFLINE_MODE'] = 'true' - os.environ['TTS_ASR_MPS_MEMORY_LIMIT_MB'] = '4096' - - # 重新加载 - if 'backend.tts_asr' in sys.modules: - del sys.modules['backend.tts_asr'] - - from backend.tts_asr import ( - TTS_ASR_MODEL_SIZE, TTS_ASR_QUANTIZE, - TTS_ASR_OFFLINE_MODE, TTS_ASR_MPS_MEMORY_LIMIT_MB - ) - - assert TTS_ASR_MODEL_SIZE == 'small' - assert TTS_ASR_QUANTIZE == True - assert TTS_ASR_OFFLINE_MODE == True - assert TTS_ASR_MPS_MEMORY_LIMIT_MB == 4096 - - print(f"✓ TTS_ASR_MODEL_SIZE = {TTS_ASR_MODEL_SIZE}") - print(f"✓ TTS_ASR_QUANTIZE = {TTS_ASR_QUANTIZE}") - print(f"✓ TTS_ASR_OFFLINE_MODE = {TTS_ASR_OFFLINE_MODE}") - print(f"✓ TTS_ASR_MPS_MEMORY_LIMIT_MB = {TTS_ASR_MPS_MEMORY_LIMIT_MB}") - - print("\n✓ 测试通过") - return True - - finally: - # 恢复原始值 - for var, value in original_values.items(): - if value is not None: - os.environ[var] = value - elif var in os.environ: - del os.environ[var] - - -def run_full_simulation(): - """运行完整模拟测试""" - print("\n" + "="*70) - print("完整macOS环境模拟测试") - print("="*70) - - results = [] - - # 运行所有测试 - results.append(("设备检测", test_device_detection_on_apple_silicon())) - results.append(("内存管理", test_memory_management())) - results.append(("模型选择", test_model_size_selection())) - results.append(("音频处理", test_audio_processing())) - results.append(("环境变量", test_environment_variables())) - - # 汇总结果 - print("\n" + "="*70) - print("测试结果汇总") - print("="*70) - - for name, passed in results: - status = "✓ 通过" if passed else "✗ 失败" - print(f"{name}: {status}") - - total = len(results) - passed = sum(1 for _, p in results if p) - - print("\n" + "-"*70) - print(f"总计: {passed}/{total} 测试通过") - print("="*70) - - return all(p for _, p in results) - - -def main(): - parser = argparse.ArgumentParser( - description='macOS环境模拟测试工具', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -示例: - # 运行完整模拟测试 - python backend/tests/simulate_macos.py --full-simulation - - # 仅模拟Apple Silicon环境 - python backend/tests/simulate_macos.py --apple-silicon - - # 仅模拟MPS设备 - python backend/tests/simulate_macos.py --device mps - - # 仅模拟CUDA设备 - python backend/tests/simulate_macos.py --device cuda - """ - ) - - parser.add_argument( - '--full-simulation', - action='store_true', - help='运行完整模拟测试' - ) - - parser.add_argument( - '--apple-silicon', - action='store_true', - help='模拟Apple Silicon环境' - ) - - parser.add_argument( - '--device', - choices=['mps', 'cuda'], - help='模拟特定设备' - ) - - parser.add_argument( - '--test', - choices=['device', 'memory', 'model', 'audio', 'env'], - help='运行特定测试' - ) - - args = parser.parse_args() - - # 确保可以导入backend模块 - sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) - - if args.full_simulation: - success = run_full_simulation() - sys.exit(0 if success else 1) - - if args.apple_silicon: - simulator = MacOSSimulator() - try: - simulator.simulate_apple_silicon() - simulator.simulate_mps_device() - - print("\n环境已模拟，按Ctrl+D退出") - print("在Python环境中可以使用:") - print(" from backend.tts_asr import _is_apple_silicon") - print(" print(_is_apple_silicon()) # 应该返回 True") - - # 进入交互模式 - import code - code.interact(local=locals()) - finally: - simulator.cleanup() - - if args.device: - simulator = MacOSSimulator() - try: - if args.device == 'mps': - simulator.simulate_mps_device() - elif args.device == 'cuda': - simulator.simulate_cuda_device() - - print("\n设备已模拟") - import code - code.interact(local=locals()) - finally: - simulator.cleanup() - - if args.test: - test_func = { - 'device': test_device_detection_on_apple_silicon, - 'memory': test_memory_management, - 'model': test_model_size_selection, - 'audio': test_audio_processing, - 'env': test_environment_variables, - } - - success = test_func[args.test]() - sys.exit(0 if success else 1) - - # 默认运行完整测试 - if not any([args.full_simulation, args.apple_silicon, args.device, args.test]): - parser.print_help() - - -if __name__ == '__main__': - main() diff --git a/backend/tests/test_audit_store.py b/backend/tests/test_audit_store.py new file mode 100644 index 0000000..ab8a553 --- /dev/null +++ b/backend/tests/test_audit_store.py @@ -0,0 +1,75 @@ +"""Regression tests for PostgreSQL audit persistence.""" + +from __future__ import annotations + +from pathlib import Path + + +BACKEND_DIR = Path(__file__).resolve().parents[1] +if str(BACKEND_DIR) not in __import__("sys").path: + __import__("sys").path.insert(0, str(BACKEND_DIR)) + +import audit_store # noqa: E402 +from audit_store import PostgresAuditStore # noqa: E402 + + +class _RecordingCursor: + def __init__(self) -> None: + self.query = "" + self.params = () + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, query: str, params=()) -> None: + self.query = query + self.params = params or () + assert query.count("%s") == len(self.params) + + +class _RecordingConnection: + def __init__(self, cursor: _RecordingCursor) -> None: + self._cursor = cursor + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self) -> _RecordingCursor: + return self._cursor + + +def test_record_llm_call_keeps_columns_placeholders_and_params_aligned(monkeypatch): + cursor = _RecordingCursor() + monkeypatch.setattr(audit_store, "psycopg", object()) + store = PostgresAuditStore("postgresql://unused") + store._initialized = True + monkeypatch.setattr(store, "_connect", lambda: _RecordingConnection(cursor)) + + store.record_llm_call({ + "request_id": "request-1", + "session_hash": "session", + "ip_hash": "ip", + "job_type": "ocr", + "model": "vision-model", + "estimated_input_tokens": 12, + "max_output_tokens": 256, + "estimated_cost": 0.01, + "actual_output_chars": 42, + "actual_cost": 0.02, + "queue_ms": 10, + "run_ms": 20, + "total_ms": 30, + "status": "completed", + "error_code": "", + "metadata": {"source": "test"}, + }) + + assert "INSERT INTO llm_call_audit" in cursor.query + assert cursor.query.count("%s") == 16 + assert len(cursor.params) == 16 diff --git a/backend/tests/test_main_cancel.py b/backend/tests/test_main_cancel.py index 3600966..0d3ed80 100644 --- a/backend/tests/test_main_cancel.py +++ b/backend/tests/test_main_cancel.py @@ -90,6 +90,72 @@ def test_cancel_endpoint_cancels_running_task(monkeypatch): assert "event: cancelled" in response_box["body"] +class FakeRedis: + def __init__(self): + self.acks = [] + + async def xack(self, *args): + self.acks.append(args) + + async def hincrby(self, key, field, amount): + return 0 + + +class FakeManager: + def __init__(self): + self.redis = FakeRedis() + self.statuses = {} + + async def get_status(self, job_id): + return self.statuses.get(job_id) + + async def _set_state(self, job_id, state): + self.statuses[job_id] = state + + async def _metrics(self, job_type): + return {"queued_count": 0, "running_count": 0} + + async def _emit_event(self, job_id, event, data): + self.statuses[job_id]["event"] = event + + def _metrics_key(self, job_type): + return f"metrics:{job_type}" + + def _state_key(self, job_id): + return f"state:{job_id}" + + +async def _run_cancelled_after_handler(manager, job_type): + worker = job_system.RedisWorker(manager) + await worker._run_message( + job_type, + "queue", + "group", + "msg-1", + {"job_id": "job-1"}, + asyncio.Semaphore(1), + ) + + +def test_redis_worker_acks_when_handler_returns_cancelled_state(): + async def handler(payload, emit, is_cancelled): + return {"ok": True} + + async def coro(): + manager = FakeManager() + manager.handlers = {"completion": handler} + manager.statuses["job-1"] = { + "request_id": "req-1", + "type": "completion", + "status": "running", + "created_at": 1, + } + await _run_cancelled_after_handler(manager, "completion") + assert manager.redis.acks == [("queue", "group", "msg-1")] + + asyncio.run(coro()) + + def test_cancel_not_found(): with TestClient(main.app) as client: response = client.post( diff --git a/backend/tests/test_main_endpoints.py b/backend/tests/test_main_endpoints.py index c707d21..c3af8b7 100644 --- a/backend/tests/test_main_endpoints.py +++ b/backend/tests/test_main_endpoints.py @@ -1,5 +1,6 @@ import base64 import asyncio +import base64 import importlib import os import sys @@ -209,6 +210,16 @@ def test_post_convert_unsupported_extension_returns_500(): assert "仅支持" in resp.json()["error"] +def test_post_convert_rejects_mismatched_content_suffix(): + content = base64.b64encode(b"%PDF-1.4\n%%EOF").decode() + with TestClient(main.app) as client: + resp = client.post("/v1/convert", headers=HEADERS, json={ + "file": content, "filename": "sample.txt", + }) + assert resp.status_code == 500 + assert "仅支持" in resp.json()["error"] + + def test_docs_nodes_crud_round_trip(): with TestClient(main.app) as client: folder_resp = client.post("/v1/docs/folders", headers=HEADERS, json={ diff --git a/backend/tests/test_tts_asr.py b/backend/tests/test_tts_asr.py new file mode 100644 index 0000000..0412d06 --- /dev/null +++ b/backend/tests/test_tts_asr.py @@ -0,0 +1,334 @@ +"""Tests for the shared LLM speech adapter and speech job handlers.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import tempfile +from pathlib import Path + +import httpx +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +BACKEND_DIR = Path(__file__).resolve().parents[1] +if str(BACKEND_DIR) not in __import__("sys").path: + __import__("sys").path.insert(0, str(BACKEND_DIR)) + +import job_handlers # noqa: E402 +import tts_asr # noqa: E402 +from audit_store import BaseAuditStore # noqa: E402 + + +def _wav_bytes(duration_ms: int = 100) -> bytes: + sample_rate = 16000 + frames = max(1, int(sample_rate * duration_ms / 1000)) + data = b"".join((i % 32768).to_bytes(2, "little", signed=False) for i in range(frames)) + data_size = len(data) + return ( + b"RIFF" + (36 + data_size).to_bytes(4, "little") + + b"WAVE" + + b"fmt " + (16).to_bytes(4, "little") + + (1).to_bytes(2, "little") + + (1).to_bytes(2, "little") + + sample_rate.to_bytes(4, "little") + + sample_rate.to_bytes(4, "little") + + (2).to_bytes(2, "little") + + (16).to_bytes(2, "little") + + b"data" + data_size.to_bytes(4, "little") + + data + ) + + +def _run_async(coro): + return asyncio.run(coro) + + +class _CaptureAuditStore(BaseAuditStore): + def __init__(self) -> None: + self.llm_calls: list[dict] = [] + + def record_llm_call(self, payload: dict) -> None: + self.llm_calls.append(payload) + + +def test_tts_calls_shared_llm_speech_endpoint(monkeypatch): + captured: dict[str, object] = {} + monkeypatch.setattr(tts_asr, "LLM_API_KEY", "test-api-key") + monkeypatch.setattr(tts_asr, "TTS_MODEL_ID", "Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit") + + def transport(request: httpx.Request): + captured["url"] = str(request.url) + captured["headers"] = dict(request.headers) + captured["json"] = json.loads(request.read().decode("utf-8")) + return httpx.Response(200, content=b"speech-ok", headers={"x-request-id": "tts-req-1"}) + + async def run(): + client = httpx.AsyncClient( + base_url="https://speech.example/v1", + transport=httpx.MockTransport(transport), + ) + try: + tts_asr._httpx_client = client + return await tts_asr.generate_tts_response( + "你好世界", + instruct="A warm Mandarin voice.", + speaker="Vivian", + output_format="wav", + ) + finally: + await client.aclose() + tts_asr._httpx_client = None + + result = _run_async(run()) + + assert result["format"] == "wav" + assert result["speaker"] == "Vivian" + assert result["model"] == "Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit" + assert result["upstream_request_id"] == "tts-req-1" + assert base64.b64decode(result["audio_base64"]) == b"speech-ok" + assert captured["url"] == "https://speech.example/v1/audio/speech" + assert captured["headers"]["authorization"] == "Bearer test-api-key" + payload = captured["json"] + assert payload["model"] == "Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit" + assert payload["voice"] == "Vivian" + assert payload["input"] == "你好世界" + assert payload["instructions"] == "A warm Mandarin voice." + assert "instruction" not in payload + + +def test_tts_uses_nonempty_default_instructions(monkeypatch): + captured: dict[str, object] = {} + + def transport(request: httpx.Request): + captured["json"] = json.loads(request.read().decode("utf-8")) + return httpx.Response(200, content=b"speech-ok") + + async def run(): + client = httpx.AsyncClient( + base_url="https://speech.example/v1", + transport=httpx.MockTransport(transport), + ) + try: + tts_asr._httpx_client = client + return await tts_asr.generate_tts_response("你好世界") + finally: + await client.aclose() + tts_asr._httpx_client = None + + _run_async(run()) + + payload = captured["json"] + assert payload["instructions"] == tts_asr.DEFAULT_TTS_INSTRUCTIONS + assert payload["instructions"].strip() + + +def test_asr_calls_shared_llm_transcriptions_endpoint(monkeypatch): + captured: dict[str, object] = {} + monkeypatch.setattr(tts_asr, "LLM_API_KEY", "test-api-key") + monkeypatch.setattr(tts_asr, "ASR_MODEL_ID", "Qwen3-ASR-0.6B-8bit") + + def transport(request: httpx.Request): + captured["url"] = str(request.url) + captured["headers"] = dict(request.headers) + captured["content"] = request.read() + return httpx.Response(200, json={"text": "hello world", "language": "zh"}, headers={"x-request-id": "asr-req-1"}) + + async def run(): + client = httpx.AsyncClient( + base_url="https://speech.example/v1", + transport=httpx.MockTransport(transport), + ) + try: + tts_asr._httpx_client = client + return await tts_asr.generate_asr_response(_wav_bytes(), language="zh-CN") + finally: + await client.aclose() + tts_asr._httpx_client = None + + result = _run_async(run()) + + assert result["text"] == "hello world" + assert result["language"] == "zh" + assert result["model"] == "Qwen3-ASR-0.6B-8bit" + assert result["upstream_request_id"] == "asr-req-1" + assert captured["url"] == "https://speech.example/v1/audio/transcriptions" + assert captured["headers"]["authorization"] == "Bearer test-api-key" + content = captured["content"] + assert b'name="model"' in content + assert b"Qwen3-ASR-0.6B-8bit" in content + assert b'name="language"' in content + assert b"zh" in content + + +def test_invalid_tts_text_returns_http_exception(): + with pytest.raises(tts_asr.HTTPException) as exc: + _run_async(tts_asr._call_tts_api("", speaker="Vivian")) + assert exc.value.status_code == 400 + + +def test_invalid_asr_audio_returns_http_exception(): + with pytest.raises(tts_asr.HTTPException) as exc: + _run_async(tts_asr._call_asr_api(b"", language="zh-CN")) + assert exc.value.status_code == 400 + + +def test_status_config_routes(monkeypatch): + app = FastAPI() + app.include_router(tts_asr.meta_router) + monkeypatch.setattr(tts_asr, "LLM_BASE_URL", "https://speech.example/v1") + monkeypatch.setattr(tts_asr, "LLM_API_KEY", "") + monkeypatch.setattr(tts_asr, "TTS_MODEL_ID", "Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit") + monkeypatch.setattr(tts_asr, "ASR_MODEL_ID", "Qwen3-ASR-0.6B-8bit") + + with TestClient(app) as client: + status = client.get("/status") + config = client.get("/config") + + assert status.status_code == 200 + assert config.status_code == 200 + assert status.json()["llm_url"] == "https://speech.example/v1" + assert status.json()["tts_model"] == "Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit" + assert status.json()["asr_model"] == "Qwen3-ASR-0.6B-8bit" + assert status.json()["status"]["api_key_configured"] is False + assert status.json()["status"]["max_connections"] == tts_asr.SPEECH_MAX_CONNECTIONS + + +def test_tts_concurrent_requests_respect_connection_limit(monkeypatch): + monkeypatch.setattr(tts_asr, "SPEECH_MAX_CONNECTIONS", 4) + monkeypatch.setattr(tts_asr, "SPEECH_MAX_KEEPALIVE_CONNECTIONS", 1) + + class LimitedClient: + def __init__(self): + self.semaphore = asyncio.Semaphore(4) + self.active = 0 + self.max_active = 0 + + async def post(self, url: str, **kwargs): + async with self.semaphore: + self.active += 1 + self.max_active = max(self.max_active, self.active) + await asyncio.sleep(0.01) + self.active -= 1 + return httpx.Response(200, content=b"speech-ok", request=httpx.Request("POST", f"https://speech.example{url}")) + + async def run(): + client = LimitedClient() + + async def get_client(): + return client + + monkeypatch.setattr(tts_asr, "_get_speech_client", get_client) + await asyncio.gather(*(tts_asr.generate_tts_response(f"文本 {index}") for index in range(20))) + return client + + client = _run_async(run()) + assert client.max_active <= 4 + + +def test_tts_asr_handlers_record_audit(monkeypatch): + audit_store = _CaptureAuditStore() + + async def fake_tts(*args, **kwargs): + return { + "audio_base64": base64.b64encode(b"ok").decode("utf-8"), + "format": "wav", + "duration_ms": 1200, + "audio_bytes": 2, + "text_chars": 2, + "speaker": "Vivian", + "model": "Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit", + "request_ms": 45, + "upstream_request_id": "tts-upstream", + } + + async def fake_asr(*args, **kwargs): + return { + "text": "hello world", + "language": "zh", + "audio_bytes": len(_wav_bytes()), + "model": "Qwen3-ASR-0.6B-8bit", + "request_ms": 80, + "upstream_request_id": "asr-upstream", + } + + monkeypatch.setattr(job_handlers, "generate_tts_response", fake_tts) + monkeypatch.setattr(job_handlers, "generate_asr_response", fake_asr) + monkeypatch.setattr(job_handlers, "get_audit_store", lambda *_args, **_kwargs: audit_store) + + base_payload = { + "request_id": "req-1", + "risk": { + "request_id": "req-1", + "session_hash": "session", + "ip_hash": "ip", + "estimated_input_tokens": 12, + "estimated_cost": 0.0, + "policy": { + "job_type": "tts", + "model": "Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit", + "profile": "speech_tts", + "max_output_tokens": 0, + }, + }, + "job_context": { + "created_at": 1000, + "started_at": 1200, + "queue_ms": 200, + }, + } + + async def run(): + events = [] + + async def emit(event: str, data: dict): + events.append((event, data)) + + tts_payload = { + **base_payload, + "text": "你好", + "speaker": "Vivian", + "format": "wav", + } + await job_handlers.tts_handler(tts_payload, emit, lambda: False) + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as handle: + handle.write(_wav_bytes()) + audio_path = handle.name + + try: + asr_payload = { + **base_payload, + "risk": { + **base_payload["risk"], + "policy": { + "job_type": "asr", + "model": "Qwen3-ASR-0.6B-8bit", + "profile": "speech_asr", + "max_output_tokens": 0, + }, + }, + "input_path": audio_path, + "language": "zh-CN", + } + await job_handlers.asr_handler(asr_payload, emit, lambda: False) + finally: + job_handlers._safe_unlink(audio_path) + + return events + + events = _run_async(run()) + + assert any(event == "result" for event, _data in events) + assert len(audit_store.llm_calls) == 2 + tts_audit = audit_store.llm_calls[0] + asr_audit = audit_store.llm_calls[1] + assert tts_audit["job_type"] == "tts" + assert tts_audit["queue_ms"] == 200 + assert tts_audit["metadata"]["duration_ms"] == 1200 + assert tts_audit["metadata"]["upstream_request_id"] == "tts-upstream" + assert asr_audit["job_type"] == "asr" + assert asr_audit["metadata"]["language"] == "zh" + assert asr_audit["metadata"]["upstream_request_id"] == "asr-upstream" diff --git a/backend/tests/test_web_search.py b/backend/tests/test_web_search.py index bd5a413..39d2400 100644 --- a/backend/tests/test_web_search.py +++ b/backend/tests/test_web_search.py @@ -1,5 +1,6 @@ import asyncio import importlib +import socket import os import sys import threading @@ -44,12 +45,26 @@ def _payload(): } -def test_is_blocked_public_url(): +def test_is_blocked_public_url(monkeypatch): + def fake_getaddrinfo(host, port, type=0, flags=0): # noqa: ARG001 + del host, flags + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", port, 0, 0))] + + monkeypatch.setattr(job_handlers.socket, "getaddrinfo", fake_getaddrinfo) assert job_handlers._is_blocked_public_url("http://127.0.0.1/test") is True assert job_handlers._is_blocked_public_url("file:///tmp/test") is True assert job_handlers._is_blocked_public_url("https://example.com/docs") is False +def test_is_blocked_public_url_resolves_private_hostname(monkeypatch): + def fake_getaddrinfo(host, port, type=0, flags=0): # noqa: ARG001 + del host, flags + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", port, 0, 0))] + + monkeypatch.setattr(job_handlers.socket, "getaddrinfo", fake_getaddrinfo) + assert job_handlers._is_blocked_public_url("https://private.example.com/docs") is True + + def test_web_search_route_returns_done(monkeypatch): async def fake_call_ollama(prompt, system_prompt=None, tag="", **kwargs): # noqa: ARG001 if tag.endswith("-webq"): diff --git a/backend/tts_asr.py b/backend/tts_asr.py index 359332a..4a73a61 100644 --- a/backend/tts_asr.py +++ b/backend/tts_asr.py @@ -1,174 +1,133 @@ +"""OpenAI-compatible TTS/ASR adapter bound to the shared LLM API.""" + +from __future__ import annotations + import asyncio import base64 import logging import os -import tempfile -from typing import Optional - -os.environ.setdefault("HF_ENDPOINT", "https://hf-mirror.com") +import time +from typing import Any, Optional +import httpx from fastapi import APIRouter, HTTPException from pydantic import BaseModel logger = logging.getLogger(__name__) -try: - import numpy as np # type: ignore -except Exception as exc: # pragma: no cover - logger.debug("numpy import failed: %s", exc) - np = None # type: ignore -try: - import torch # type: ignore -except Exception as exc: # pragma: no cover - logger.debug("torch import failed: %s", exc) - torch = None # type: ignore +def _int_env(name: str, default: int) -> int: + try: + return max(1, int(os.getenv(name, str(default)))) + except (TypeError, ValueError): + return default -try: - from qwen_tts import Qwen3TTSModel # type: ignore -except Exception as exc: # pragma: no cover - logger.debug("qwen_tts import failed: %s", exc) - Qwen3TTSModel = None # type: ignore - -try: - from faster_whisper import WhisperModel # type: ignore -except Exception as exc: # pragma: no cover - logger.debug("faster_whisper import failed: %s", exc) - WhisperModel = None # type: ignore - -try: - from modelscope import snapshot_download # type: ignore -except Exception as exc: # pragma: no cover - logger.debug("modelscope import failed: %s", exc) - snapshot_download = None # type: ignore meta_router = APIRouter() -generation_router = APIRouter() -MODEL_ID_HF = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign" -MODEL_ID_MS = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign" -ASR_MODEL_ID = os.getenv("ASR_MODEL_ID", "small") -ASR_COMPUTE_TYPE = os.getenv("ASR_COMPUTE_TYPE", "int8") +LLM_BASE_URL = (os.getenv("LLM_BASE_URL", "https://api.openai.com/v1/") or "").strip().rstrip("/") +LLM_API_KEY = (os.getenv("LLM_API_KEY", "") or "").strip() -_tts_model: Optional["Qwen3TTSModel"] = None -_asr_model: Optional["WhisperModel"] = None +DEFAULT_TTS_MODEL_ID = "Qwen3-TTS-12Hz-1.7B-VoiceDesign-8bit" +DEFAULT_ASR_MODEL_ID = "Qwen3-ASR-0.6B-8bit" +DEFAULT_TTS_INSTRUCTIONS = ( + os.getenv("TTS_DEFAULT_INSTRUCTIONS", "A clear, natural voice speaking Mandarin Chinese.") + or "A clear, natural voice speaking Mandarin Chinese." +).strip() + +TTS_MODEL_ID = (os.getenv("TTS_MODEL_ID", DEFAULT_TTS_MODEL_ID) or DEFAULT_TTS_MODEL_ID).strip() +ASR_MODEL_ID = (os.getenv("ASR_MODEL_ID", DEFAULT_ASR_MODEL_ID) or DEFAULT_ASR_MODEL_ID).strip() + +TTS_MAX_TEXT_CHARS = _int_env("TTS_ASR_MAX_TEXT_CHARS", 4096) +ASR_MAX_AUDIO_BYTES = _int_env("ASR_MAX_AUDIO_BYTES", 100 * 1024 * 1024) +TTS_TIMEOUT_SECONDS = _int_env("TTS_ASR_TTS_TIMEOUT_SECONDS", 180) +ASR_TIMEOUT_SECONDS = _int_env("TTS_ASR_ASR_TIMEOUT_SECONDS", 300) +HEALTHCHECK_TIMEOUT_SECONDS = _int_env("TTS_ASR_HEALTHCHECK_TIMEOUT_SECONDS", 5) +SPEECH_MAX_CONNECTIONS = _int_env("TTS_ASR_MAX_CONNECTIONS", 16) +SPEECH_MAX_KEEPALIVE_CONNECTIONS = _int_env("TTS_ASR_MAX_KEEPALIVE_CONNECTIONS", 8) + +_httpx_client: Optional[httpx.AsyncClient] = None +_httpx_client_lock = asyncio.Lock() -def _get_device_map() -> str: - if torch is None: - return "cpu" - if torch.cuda.is_available(): - return "cuda" - try: - if hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): - return "mps" - except Exception as exc: # pragma: no cover - logger.debug("MPS check failed: %s", exc) - return "cpu" - - -def _download_tts_model_from_modelscope() -> Optional[str]: - if snapshot_download is None: +def _read_uint16(data: bytes, offset: int) -> Optional[int]: + if len(data) < offset + 2: return None - cache_dir = os.path.join(os.path.dirname(__file__), "models") - os.makedirs(cache_dir, exist_ok=True) - try: - return snapshot_download(MODEL_ID_MS, cache_dir=cache_dir, revision="master") - except Exception as exc: # pragma: no cover - logger.warning("ModelScope TTS download failed: %s", exc) + return int.from_bytes(data[offset : offset + 2], "little", signed=False) + + +def _read_uint32(data: bytes, offset: int) -> Optional[int]: + if len(data) < offset + 4: return None + return int.from_bytes(data[offset : offset + 4], "little", signed=False) -def _ensure_tts_model() -> "Qwen3TTSModel": - global _tts_model - if _tts_model is not None: - return _tts_model - if np is None or torch is None or Qwen3TTSModel is None: - raise RuntimeError("TTS 依赖未安装完整") +def _parse_wav_duration_ms(audio_bytes: bytes) -> int: + if len(audio_bytes) < 44 or audio_bytes[:4] != b"RIFF" or audio_bytes[8:12] != b"WAVE": + return 0 - device_map = _get_device_map() - dtype = torch.float16 if device_map != "cpu" else torch.float32 + data_size = 0 + byte_rate = 0 + offset = 12 - model_path = _download_tts_model_from_modelscope() - last_error = None + while offset + 8 <= len(audio_bytes): + chunk_id = audio_bytes[offset : offset + 4] + chunk_size = _read_uint32(audio_bytes, offset + 4) + if chunk_size is None: + break + chunk_start = offset + 8 + chunk_end = min(chunk_start + chunk_size, len(audio_bytes)) - for candidate in [model_path, MODEL_ID_HF]: - if not candidate: - continue - try: - _tts_model = Qwen3TTSModel.from_pretrained( # type: ignore - candidate, - device_map=device_map, - dtype=dtype, - ) - return _tts_model - except Exception as exc: - last_error = exc - logger.warning("TTS model load failed from %s: %s", candidate, exc) + if chunk_id == b"fmt ": + audio_format = _read_uint16(audio_bytes, chunk_start) + channels = _read_uint16(audio_bytes, chunk_start + 2) + sample_rate = _read_uint32(audio_bytes, chunk_start + 4) + bits_per_sample = _read_uint16(audio_bytes, chunk_start + 14) + if audio_format == 1 and channels and sample_rate and bits_per_sample: + byte_rate = int(sample_rate * channels * bits_per_sample // 8) - raise RuntimeError(f"TTS 模型加载失败: {last_error}") from last_error + if chunk_id == b"data": + data_size = chunk_size + offset = chunk_end + (chunk_end - chunk_start) % 2 + + if data_size and byte_rate: + return max(0, int(data_size * 1000 / byte_rate)) + return 0 -def _ensure_asr_model() -> "WhisperModel": - global _asr_model - if _asr_model is not None: - return _asr_model - if WhisperModel is None: - raise RuntimeError("faster-whisper 未安装") - - device = "cuda" if _get_device_map() == "cuda" else "cpu" - compute_type = ASR_COMPUTE_TYPE if device == "cpu" else "float16" - _asr_model = WhisperModel(ASR_MODEL_ID, device=device, compute_type=compute_type) - return _asr_model +def _duration_from_audio_bytes(audio_bytes: bytes) -> int: + return _parse_wav_duration_ms(audio_bytes) -async def _warmup_tts(): - await asyncio.to_thread(_ensure_tts_model) +def _audio_bytes_to_base64(audio_bytes: bytes) -> str: + return base64.b64encode(audio_bytes).decode("utf-8") -async def _warmup_asr(): - await asyncio.to_thread(_ensure_asr_model) +def _normalize_tts_text(text: str) -> str: + value = (text or "").strip() + if not value: + raise HTTPException(status_code=400, detail="TTS 文本为空") + if len(value) > TTS_MAX_TEXT_CHARS: + raise HTTPException(status_code=400, detail=f"TTS 文本过长，超过限制 {TTS_MAX_TEXT_CHARS} 个字符") + return value -class TTSRequest(BaseModel): - text: str - instruct: str = "" - speaker: str = "Vivian" - format: str = "wav" +def _normalize_output_format(output_format: str) -> str: + value = (output_format or "wav").strip().lower() + if value not in {"wav", "mp3"}: + raise HTTPException(status_code=400, detail="不支持的 TTS 输出格式") + return value -class TTSResponse(BaseModel): - audio_base64: str - format: str - duration_ms: int - - -class ASRRequest(BaseModel): - audio_base64: str - language: Optional[str] = "zh-CN" - - -class ASRResponse(BaseModel): - text: str - language: Optional[str] = None - - -class ModelStatus(BaseModel): - tts_loaded: bool - asr_loaded: bool = False - device: str - - -def _normalize_language(language: Optional[str]) -> Optional[str]: +def _normalize_asr_language(language: Optional[str]) -> Optional[str]: if not language: return None - value = language.strip().lower() - if value in {"auto", ""}: + value = str(language).strip().lower() + if value in {"", "auto"}: return None mapping = { "zh-cn": "zh", "zh-hans": "zh", - "zh-tw": "zh", "en-us": "en", "ja-jp": "ja", "ko-kr": "ko", @@ -176,38 +135,138 @@ def _normalize_language(language: Optional[str]) -> Optional[str]: return mapping.get(value, value.split("-")[0]) -@meta_router.get("/status", response_model=ModelStatus) -async def get_status(): - return ModelStatus( - tts_loaded=_tts_model is not None, - asr_loaded=_asr_model is not None, - device=_get_device_map(), - ) +def _speech_headers() -> dict[str, str]: + headers = {"Accept": "*/*"} + if LLM_API_KEY: + headers["Authorization"] = f"Bearer {LLM_API_KEY}" + headers["X-API-Key"] = LLM_API_KEY + return headers -@meta_router.get("/config") -async def get_config(): +def _raise_http_error(response: httpx.Response, operation: str) -> None: + try: + response.raise_for_status() + except httpx.HTTPStatusError as exc: + body = (exc.response.text or "").strip()[:1000] + detail = f"{operation} 请求失败 HTTP {exc.response.status_code}" + if body: + detail = f"{detail}: {body}" + raise HTTPException(status_code=exc.response.status_code, detail=detail) from exc + except Exception as exc: + raise HTTPException(status_code=502, detail=f"{operation} 请求失败: {exc}") from exc + + +def _tts_timeout() -> httpx.Timeout: + return httpx.Timeout(TTS_TIMEOUT_SECONDS, connect=5.0) + + +def _asr_timeout() -> httpx.Timeout: + return httpx.Timeout(ASR_TIMEOUT_SECONDS, connect=5.0) + + +def _extract_upstream_request_id(response: httpx.Response) -> str: + for header_name in ("x-request-id", "request-id", "openai-request-id"): + value = (response.headers.get(header_name) or "").strip() + if value: + return value + return "" + + +async def _get_speech_client() -> httpx.AsyncClient: + global _httpx_client + + if _httpx_client is None or getattr(_httpx_client, "is_closed", False): + limits = httpx.Limits( + max_connections=SPEECH_MAX_CONNECTIONS, + max_keepalive_connections=max(1, SPEECH_MAX_KEEPALIVE_CONNECTIONS), + ) + async with _httpx_client_lock: + if _httpx_client is None or getattr(_httpx_client, "is_closed", False): + _httpx_client = httpx.AsyncClient( + base_url=LLM_BASE_URL, + timeout=_tts_timeout(), + headers=_speech_headers(), + follow_redirects=True, + limits=limits, + ) + return _httpx_client + + +async def close_speech_client() -> None: + global _httpx_client + if _httpx_client is not None and not getattr(_httpx_client, "is_closed", False): + await _httpx_client.aclose() + _httpx_client = None + + +async def _call_tts_api(text: str, instruct: str = "", speaker: str = "Vivian", output_format: str = "wav") -> dict[str, Any]: + normalized_text = _normalize_tts_text(text) + normalized_format = _normalize_output_format(output_format) + client = await _get_speech_client() + + payload: dict[str, Any] = { + "model": TTS_MODEL_ID, + "input": normalized_text, + "response_format": normalized_format, + "voice": speaker or "Vivian", + } + payload["instructions"] = (instruct or "").strip() or DEFAULT_TTS_INSTRUCTIONS + + started_at = time.perf_counter() + response = await client.post("audio/speech", json=payload, timeout=_tts_timeout(), headers=_speech_headers()) + elapsed_ms = int((time.perf_counter() - started_at) * 1000) + _raise_http_error(response, "TTS") + audio_bytes = response.content + if not audio_bytes: + raise HTTPException(status_code=502, detail="TTS API 返回音频为空") return { - "model": { - "tts": MODEL_ID_MS, - "asr": ASR_MODEL_ID, - }, - "device": _get_device_map(), - "status": { - "tts_loaded": _tts_model is not None, - "asr_loaded": _asr_model is not None, - } + "audio_bytes": audio_bytes, + "request_ms": elapsed_ms, + "upstream_request_id": _extract_upstream_request_id(response), } -@meta_router.post("/warmup") -async def warmup_models(): - await _warmup_tts() - await _warmup_asr() +async def _call_asr_api(audio_bytes: bytes, language: Optional[str] = "zh-CN") -> dict[str, Any]: + if not audio_bytes: + raise HTTPException(status_code=400, detail="ASR 音频内容为空") + if len(audio_bytes) > ASR_MAX_AUDIO_BYTES: + raise HTTPException(status_code=400, detail=f"ASR 音频过大，超过限制 {ASR_MAX_AUDIO_BYTES} 字节") + + normalized_language = _normalize_asr_language(language) + client = await _get_speech_client() + files = {"file": ("audio.wav", audio_bytes, "audio/wav")} + data = {"model": ASR_MODEL_ID} + if normalized_language: + data["language"] = normalized_language + + started_at = time.perf_counter() + response = await client.post( + "audio/transcriptions", + files=files, + data=data, + timeout=_asr_timeout(), + headers=_speech_headers(), + ) + elapsed_ms = int((time.perf_counter() - started_at) * 1000) + _raise_http_error(response, "ASR") + try: + result = response.json() + except ValueError as exc: + raise HTTPException(status_code=502, detail="ASR API 返回非 JSON 数据") from exc + + if not isinstance(result, dict): + raise HTTPException(status_code=502, detail="ASR API 返回结构异常") + + text = str(result.get("text", "") or "").strip() + if not text: + raise HTTPException(status_code=422, detail="ASR API 返回结果为空") + + detected_language = result.get("language") or normalized_language or "auto" return { - "tts_warmup": _tts_model is not None, - "asr_warmup": _asr_model is not None, - "device": _get_device_map(), + "text": text, + "language": str(detected_language), + "request_ms": elapsed_ms, + "upstream_request_id": _extract_upstream_request_id(response), } @@ -216,113 +275,97 @@ async def generate_tts_response( instruct: str = "", speaker: str = "Vivian", output_format: str = "wav", -) -> TTSResponse: - del speaker - del output_format - if np is None: - raise HTTPException(status_code=501, detail="numpy 未安装，TTS 功能不可用") - - try: - model = _ensure_tts_model() - except Exception as exc: - raise HTTPException(status_code=500, detail=str(exc)) - - try: - wavs, sample_rate = await asyncio.to_thread( - model.generate_voice_design, # type: ignore - text=text, - language="Chinese", - instruct=instruct or "", - ) - except Exception as exc: - logger.exception("TTS inference failed") - raise HTTPException(status_code=500, detail=f"TTS 推理失败: {exc}") - - wav_data = wavs[0] if isinstance(wavs, (list, tuple)) else wavs - if hasattr(wav_data, "cpu"): - wav_data = wav_data.cpu().numpy() - wav_data = np.asarray(wav_data, dtype=np.float32) - - tmp_path = None - try: - import soundfile as sf # type: ignore - - fd, tmp_path = tempfile.mkstemp(suffix=".wav") - os.close(fd) - sf.write(tmp_path, wav_data, sample_rate) - with open(tmp_path, "rb") as handle: - audio_bytes = handle.read() - except Exception as exc: - logger.exception("TTS audio encode failed") - raise HTTPException(status_code=500, detail=f"音频编码失败: {exc}") - finally: - if tmp_path and os.path.exists(tmp_path): - os.unlink(tmp_path) - - duration_ms = int(len(wav_data) / sample_rate * 1000) if sample_rate > 0 else 0 - return TTSResponse( - audio_base64=base64.b64encode(audio_bytes).decode("utf-8"), - format="wav", - duration_ms=duration_ms, +) -> dict[str, Any]: + result = await _call_tts_api( + text=text, + instruct=instruct or "", + speaker=speaker or "Vivian", + output_format=output_format or "wav", ) + audio_bytes = bytes(result["audio_bytes"]) + return { + "audio_base64": _audio_bytes_to_base64(audio_bytes), + "format": _normalize_output_format(output_format or "wav"), + "duration_ms": _duration_from_audio_bytes(audio_bytes), + "audio_bytes": len(audio_bytes), + "text_chars": len(_normalize_tts_text(text)), + "speaker": speaker or "Vivian", + "model": TTS_MODEL_ID, + "request_ms": int(result.get("request_ms", 0) or 0), + "upstream_request_id": str(result.get("upstream_request_id", "") or ""), + } -async def generate_asr_response(audio_bytes: bytes, language: Optional[str] = "zh-CN") -> ASRResponse: - if not audio_bytes: - raise HTTPException(status_code=400, detail="音频内容为空") - - try: - model = _ensure_asr_model() - except Exception as exc: - raise HTTPException(status_code=500, detail=f"ASR 模型加载失败: {exc}") - - normalized_language = _normalize_language(language) - tmp_path = None - try: - fd, tmp_path = tempfile.mkstemp(suffix=".wav") - os.close(fd) - with open(tmp_path, "wb") as handle: - handle.write(audio_bytes) - - segments, info = await asyncio.to_thread( - model.transcribe, - tmp_path, - language=normalized_language, - vad_filter=True, - beam_size=5, - ) - text = "".join(segment.text for segment in segments).strip() - if not text: - raise RuntimeError("ASR 返回结果为空") - detected_language = getattr(info, "language", normalized_language or "unknown") - return ASRResponse(text=text, language=str(detected_language)) - except HTTPException: - raise - except Exception as exc: - logger.exception("ASR inference failed") - raise HTTPException(status_code=500, detail=f"ASR 推理失败: {exc}") - finally: - if tmp_path and os.path.exists(tmp_path): - os.unlink(tmp_path) +async def generate_asr_response(audio_bytes: bytes, language: Optional[str] = "zh-CN") -> dict[str, Any]: + result = await _call_asr_api(bytes(audio_bytes or b""), language or "zh-CN") + return { + "text": str(result["text"]), + "language": str(result["language"]), + "audio_bytes": len(audio_bytes or b""), + "model": ASR_MODEL_ID, + "request_ms": int(result.get("request_ms", 0) or 0), + "upstream_request_id": str(result.get("upstream_request_id", "") or ""), + } -@generation_router.post("/tts", response_model=TTSResponse) -async def tts_endpoint(req: TTSRequest): - return await generate_tts_response( - text=req.text, - instruct=req.instruct or "", - speaker=req.speaker, - output_format=req.format, - ) +class TTSResponse(BaseModel): + audio_base64: str = "" + format: str = "wav" + duration_ms: int = 0 + audio_bytes: int = 0 + text_chars: int = 0 + speaker: str = "Vivian" + model: str = TTS_MODEL_ID + request_ms: int = 0 + upstream_request_id: str = "" -@generation_router.post("/asr", response_model=ASRResponse) -async def asr_endpoint(req: ASRRequest): - audio_bytes = base64.b64decode(req.audio_base64) - return await generate_asr_response(audio_bytes, req.language if req.language else None) +class ASRResponse(BaseModel): + text: str = "" + language: Optional[str] = None + audio_bytes: int = 0 + model: str = ASR_MODEL_ID + request_ms: int = 0 + upstream_request_id: str = "" -def register_tts_asr_routes(app, include_generation_routes: bool = True): +class ModelStatus(BaseModel): + llm_url: str + tts_model: str + asr_model: str + status: dict[str, Any] + + +def _status_payload() -> dict[str, Any]: + return { + "llm_url": LLM_BASE_URL or "", + "tts_model": TTS_MODEL_ID, + "asr_model": ASR_MODEL_ID, + "status": { + "api_configured": bool(LLM_BASE_URL), + "api_key_configured": bool(LLM_API_KEY), + "tts_model": TTS_MODEL_ID, + "asr_model": ASR_MODEL_ID, + "tts_timeout_seconds": TTS_TIMEOUT_SECONDS, + "asr_timeout_seconds": ASR_TIMEOUT_SECONDS, + "healthcheck_timeout_seconds": HEALTHCHECK_TIMEOUT_SECONDS, + "max_connections": SPEECH_MAX_CONNECTIONS, + "keepalive_connections": max(1, SPEECH_MAX_KEEPALIVE_CONNECTIONS), + "max_tts_text_chars": TTS_MAX_TEXT_CHARS, + "max_asr_audio_bytes": ASR_MAX_AUDIO_BYTES, + }, + } + + +@meta_router.get("/status", response_model=ModelStatus) +async def get_status(): + return _status_payload() + + +@meta_router.get("/config") +async def get_config(): + return _status_payload() + + +def register_tts_asr_routes(app) -> None: app.include_router(meta_router, prefix="/v1/tts-asr") - if include_generation_routes: - app.include_router(generation_router, prefix="/v1/tts-asr") diff --git a/docker-compose.yml b/docker-compose.yml index a7f10b6..1b7ad67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,25 +5,48 @@ services: dockerfile: Dockerfile.frontend args: DOCKER_REGISTRY_PREFIX: ${DOCKER_REGISTRY_PREFIX:-} + cache_from: + - type=local,src=./docker-data/build-cache/frontend + cache_to: + - type=local,dest=./docker-data/build-cache/frontend,mode=max depends_on: - - api + api: + condition: service_started + restart: unless-stopped ports: - "8080:80" + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/ || exit 1"] + interval: 30s + timeout: 5s + retries: 3 postgres: image: ${DOCKER_REGISTRY_PREFIX:-}postgres:16-alpine + restart: unless-stopped environment: POSTGRES_DB: ${POSTGRES_DB:-llm_in_text} POSTGRES_USER: ${POSTGRES_USER:-llm_in_text} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-llm_in_text_change_me} volumes: - ./docker-data/postgres:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-llm_in_text} -d ${POSTGRES_DB:-llm_in_text}"] + interval: 15s + timeout: 5s + retries: 5 redis: image: ${DOCKER_REGISTRY_PREFIX:-}redis:7-alpine + restart: unless-stopped command: ["redis-server", "--appendonly", "yes"] volumes: - ./docker-data/redis:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 15s + timeout: 5s + retries: 5 searxng: image: ${DOCKER_REGISTRY_PREFIX:-}searxng/searxng:latest @@ -86,6 +109,10 @@ services: dockerfile: backend/Dockerfile args: DOCKER_REGISTRY_PREFIX: ${DOCKER_REGISTRY_PREFIX:-} + cache_from: + - type=local,src=./docker-data/build-cache/api + cache_to: + - type=local,dest=./docker-data/build-cache/api,mode=max env_file: - backend/.env environment: @@ -96,15 +123,29 @@ services: JOB_SHARED_TEMP_DIR: /shared-jobs SEARXNG_BASE_URL: http://searxng:8080 FIRECRAWL_BASE_URL: http://firecrawl:3002 + restart: unless-stopped + init: true + extra_hosts: + - "host.docker.internal:host-gateway" depends_on: - - postgres - - redis - - searxng - - firecrawl + postgres: + condition: service_healthy + redis: + condition: service_healthy + searxng: + condition: service_started + firecrawl: + condition: service_started ports: - "8001:8001" volumes: - ./docker-data/jobs:/shared-jobs + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8001/v1/tts-asr/status', timeout=5)"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 20s worker: build: @@ -112,6 +153,10 @@ services: dockerfile: backend/Dockerfile args: DOCKER_REGISTRY_PREFIX: ${DOCKER_REGISTRY_PREFIX:-} + cache_from: + - type=local,src=./docker-data/build-cache/worker + cache_to: + - type=local,dest=./docker-data/build-cache/worker,mode=max command: ["python", "worker.py"] env_file: - backend/.env @@ -123,10 +168,18 @@ services: JOB_SHARED_TEMP_DIR: /shared-jobs SEARXNG_BASE_URL: http://searxng:8080 FIRECRAWL_BASE_URL: http://firecrawl:3002 + restart: unless-stopped + init: true + extra_hosts: + - "host.docker.internal:host-gateway" depends_on: - - postgres - - redis - - searxng - - firecrawl + postgres: + condition: service_healthy + redis: + condition: service_healthy + searxng: + condition: service_started + firecrawl: + condition: service_started volumes: - ./docker-data/jobs:/shared-jobs diff --git a/docker/nginx.conf b/docker/nginx.conf index eeebea0..3aa17e6 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -8,8 +8,4 @@ server { location / { try_files $uri $uri/ /index.html; } - - location /v1/ { - return 307 https://api.imageteach.tech:8002$request_uri; - } } diff --git a/src/App.vue b/src/App.vue index 4dfc36f..ed26a7b 100644 --- a/src/App.vue +++ b/src/App.vue @@ -53,7 +53,8 @@ const backgroundStyle = computed(() => { .app-shell { position: relative; width: 100%; - height: 100vh; + height: 100dvh; + min-height: 0; background: var(--app-bg); color: var(--app-text); transition: background 0.3s, color 0.3s; diff --git a/src/components/DocBlockCrepe.vue b/src/components/DocBlockCrepe.vue index c9af312..532f324 100644 --- a/src/components/DocBlockCrepe.vue +++ b/src/components/DocBlockCrepe.vue @@ -423,11 +423,16 @@ onUnmounted(() => { } .doc-card__editor { - min-height: 48px; + min-height: 0; + height: auto; + max-height: none; + overflow: auto; + overscroll-behavior-y: contain; + scrollbar-width: thin; + scrollbar-color: var(--scrollbar-thumb) transparent; border-radius: 8px; border: 1px solid rgba(59, 130, 246, 0.08); background: rgba(255, 255, 255, 0.8); - overflow: hidden; } :root[data-theme='dark'] .doc-card__editor { @@ -437,16 +442,22 @@ onUnmounted(() => { .doc-card__editor :deep(.milkdown) { background: transparent !important; + min-height: 0; + height: auto !important; } .doc-card__editor :deep(.milkdown__main), .doc-card__editor :deep(.milkdown__editor) { margin: 0 !important; padding: 0 !important; + min-height: 0; + height: auto !important; } .doc-card__editor :deep(.ProseMirror) { min-height: 0; + height: auto !important; + overflow-x: hidden; padding: 10px 12px 12px !important; font-size: 13px !important; line-height: 1.6; @@ -456,10 +467,29 @@ onUnmounted(() => { margin-bottom: 0; } +.doc-card__editor :deep(.ProseMirror img) { + max-width: min(100%, 520px); + height: auto; +} + .doc-card__editor :deep(.ProseMirror p:first-child) { margin-top: 0; } +.doc-card__editor :deep(.cm-scroller) { + overflow-x: hidden; + overflow-y: auto; + overscroll-behavior-y: contain; + scrollbar-width: thin; + scrollbar-color: var(--scrollbar-thumb) transparent; +} + +.doc-card__editor :deep(.cm-editor) { + min-height: 0; + height: 100%; + overflow: hidden; +} + .doc-card__editor :deep(.milkdown__toolbar), .doc-card__editor :deep(.milkdown__menu), .doc-card__editor :deep(.milkdown__statusbar), diff --git a/src/components/FileContent.vue b/src/components/FileContent.vue index 0e89b38..160aa61 100644 --- a/src/components/FileContent.vue +++ b/src/components/FileContent.vue @@ -550,6 +550,8 @@ function downloadFile() { flex: 1; min-height: 0; overflow: auto; + overscroll-behavior-y: contain; + -webkit-overflow-scrolling: touch; } .directory-shell { @@ -777,6 +779,7 @@ function downloadFile() { .content-markdown { padding: 24px; + overflow-y: auto; } .markdown-body { @@ -837,6 +840,8 @@ function downloadFile() { margin: 0; padding: 18px 20px; overflow: auto; + overflow-y: auto; + overflow-x: hidden; white-space: pre; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 13px; @@ -846,6 +851,7 @@ function downloadFile() { .content-preview { padding: 20px; + overflow: auto; } .preview-surface { @@ -986,6 +992,7 @@ function downloadFile() { .pdf-frame { width: 100%; min-height: 78vh; + height: 100%; border: 1px solid var(--github-border); border-radius: 12px; background: var(--github-bg); diff --git a/src/components/FileTree.vue b/src/components/FileTree.vue index 02f5312..ea0b6b7 100644 --- a/src/components/FileTree.vue +++ b/src/components/FileTree.vue @@ -439,6 +439,8 @@ function forwardDragOver(event, id) { flex: 1; min-height: 0; overflow-y: auto; + overflow-x: hidden; + overscroll-behavior-y: contain; padding: 6px 0 10px; } @@ -648,4 +650,11 @@ function forwardDragOver(event, id) { background: rgba(99, 110, 123, 0.28); border-radius: 999px; } + +@media (max-width: 768px) { + .tree-content::-webkit-scrollbar { + width: 0 !important; + height: 0 !important; + } +} diff --git a/src/components/HiddenTextCrepe.vue b/src/components/HiddenTextCrepe.vue index 310bd6c..3a3c72e 100644 --- a/src/components/HiddenTextCrepe.vue +++ b/src/components/HiddenTextCrepe.vue @@ -166,12 +166,16 @@ watch( .hidden-text-chip__input { min-width: 4rem; + max-width: 12rem; padding: 0.08rem 0.25rem; border: 1px solid rgba(148, 163, 184, 0.45); border-radius: 0.35rem; background: rgba(255, 255, 255, 0.92); color: inherit; font: inherit; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; } .hidden-text-chip__input--visible { @@ -218,4 +222,4 @@ watch( line-height: 1; transform: translateX(0.05rem); } - \ No newline at end of file + diff --git a/src/components/ImageEditorComponent.vue b/src/components/ImageEditorComponent.vue index c4ac760..8ed65f6 100644 --- a/src/components/ImageEditorComponent.vue +++ b/src/components/ImageEditorComponent.vue @@ -194,6 +194,7 @@ defineExpose({ overflow: hidden; min-height: 640px; height: min(78vh, 920px); + height: min(78dvh, 920px); border: 1px solid var(--github-border); border-radius: 18px; background: @@ -284,6 +285,8 @@ defineExpose({ .image-editor-shell :deep(.tui-image-editor-submenu) { height: 166px; + overflow-y: auto; + overscroll-behavior-y: contain; } .image-editor-shell :deep(.tui-image-editor-submenu > div) { @@ -298,7 +301,7 @@ defineExpose({ @media (max-width: 960px) { .image-editor-shell { min-height: 560px; - height: 72vh; + height: min(72dvh, 72vh); } .image-editor-shell :deep(.tui-image-editor-main-container) { @@ -308,10 +311,11 @@ defineExpose({ .image-editor-shell :deep(.tui-image-editor-controls) { height: 88px; overflow-x: auto; + overscroll-behavior-x: contain; } .image-editor-shell :deep(.tui-image-editor-menu) { padding: 0 16px; } } - \ No newline at end of file + diff --git a/src/components/MarkdownPreview.vue b/src/components/MarkdownPreview.vue index 8703eba..e46e42b 100644 --- a/src/components/MarkdownPreview.vue +++ b/src/components/MarkdownPreview.vue @@ -71,8 +71,11 @@ const renderedContent = computed(() => { .preview-container { width: 100%; height: 100%; - padding: 20px 40px; + min-height: 0; overflow-y: auto; + overflow-x: hidden; + overscroll-behavior-y: contain; + padding: 20px 40px; background-color: #ffffff; color: #333; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif; @@ -82,7 +85,6 @@ const renderedContent = computed(() => { .preview-container :deep(.math-block) { display: block; margin: 1em 0; - text-align: center; overflow-x: auto; overflow-y: hidden; padding: 8px 0; diff --git a/src/components/MilkdownEditor.vue b/src/components/MilkdownEditor.vue index 2648904..3117667 100644 --- a/src/components/MilkdownEditor.vue +++ b/src/components/MilkdownEditor.vue @@ -31,88 +31,195 @@ -
+
+
- - - - - -
- -
- - - -
-
- - -
+
+ +
+
+
+ + + + +
-
- -
-
- -
-
-
{{ t('presetTemplates') }}
- -
- -
-
{{ t('customTemplates') }}
-

{{ t('noTemplates') }}

- -
- -
- -
-
-
- - -
@@ -382,6 +424,24 @@ const uploadButtonTitle = computed(() => { if (isDocUploadDisabled.value) return t('uploadDocInBlockWarning') || '当前光标位置不能插入文件' return t('upload') }) +const moreActionsOpen = ref(false) +const moreActionsClosing = ref(false) +const viewportWidth = ref(typeof window === 'undefined' ? 1024 : window.innerWidth) +const MORE_ACTION_MIN_HORIZONTAL_WIDTH = 520 +const MORE_ACTION_CLOSE_MS = 380 +const moreActions = computed(() => [ + { id: 'upload', ariaLabel: t('upload'), title: uploadButtonTitle.value, disabled: isDocUploadDisabled.value, disabledClass: 'force-disabled' }, + { id: 'importMd', ariaLabel: t('importMd'), title: t('importMd') }, + { id: 'exportMd', ariaLabel: t('exportMd'), title: t('exportMd') }, + { id: 'aiToggle', ariaLabel: aiButtonLabel.value, title: aiButtonLabel.value, disabled: isOverLimit.value, disabledClass: 'ai-disabled force-disabled' }, + { id: 'template', ariaLabel: t('template'), title: t('template') }, + { id: 'clear', ariaLabel: '清除', title: '清除' }, +]) +const moreActionsLayout = computed(() => ( + viewportWidth.value >= MORE_ACTION_MIN_HORIZONTAL_WIDTH ? 'horizontal' : 'vertical' +)) +const presetTemplates = computed(() => templateStore.presetTemplates || []) +const customTemplates = computed(() => templateStore.customTemplates || []) const acceptAll = computed(() => { const types = [ '.txt', '.json', '.toml', '.yaml', '.yml', @@ -404,6 +464,8 @@ let markdownSyncTimer = null let historyUpdateTimer = null let editorCopyHandler = null let documentClickHandler = null +let moreActionsCloseTimer = null +let resizeHandler = null const objectUrls = new Set() const IMAGE_NODE_TYPES = new Set(['image', 'image-block', 'imageBlock']) const MARKDOWN_EXT_RE = /\.md$/i @@ -439,6 +501,100 @@ const clearEditor = () => { } } +const finishMoreActionsClosing = () => { + if (moreActionsCloseTimer) { + clearTimeout(moreActionsCloseTimer) + moreActionsCloseTimer = null + } + moreActionsClosing.value = false +} + +const startMoreActionsClosing = () => { + finishMoreActionsClosing() + moreActionsClosing.value = true + moreActionsCloseTimer = window.setTimeout(() => { + moreActionsClosing.value = false + moreActionsCloseTimer = null + }, MORE_ACTION_CLOSE_MS) +} + +const toggleMoreActions = () => { + const nextOpen = !moreActionsOpen.value + if (nextOpen) { + finishMoreActionsClosing() + } + moreActionsOpen.value = nextOpen + showExportDropdown.value = false + showTemplateDropdown.value = false + + if (!nextOpen) { + startMoreActionsClosing() + } +} + +const closeMoreActions = () => { + if (!moreActionsOpen.value && !showExportDropdown.value && !showTemplateDropdown.value) return + moreActionsOpen.value = false + showExportDropdown.value = false + showTemplateDropdown.value = false + startMoreActionsClosing() +} + +const handleMoreActionClick = (action, event) => { + if (!moreActionsOpen.value) return + + if (action.id === 'upload') { + triggerUpload() + } else if (action.id === 'importMd') { + triggerImportMd() + } else if (action.id === 'exportMd') { + if (showExportDropdown.value) { + closeMoreActions() + } else { + toggleExportDropdown() + } + } else if (action.id === 'aiToggle') { + toggleAI() + } else if (action.id === 'template') { + if (showTemplateDropdown.value) { + closeMoreActions() + } else { + toggleTemplateDropdown() + } + } else if (action.id === 'clear') { + clearEditor() + } + + if (!['exportMd', 'template'].includes(action.id)) { + closeMoreActions() + } + + event?.stopPropagation?.() +} + +const getMoreActionStyle = (index) => { + return { + '--more-action-delay': `${index * 45}ms`, + } +} + +const isMoreActionTarget = (target) => { + if (!(target instanceof Element)) return false + return Boolean( + target.closest('.more-actions') || + target.closest('.export-btn-wrapper') || + target.closest('.template-btn-wrapper') || + target.closest('.more-actions__toggle') || + target.closest('.size-indicator') + ) +} + +const handleMoreActionKeydown = (event) => { + if (event.key === 'Escape') { + closeMoreActions() + } +} + const isPresetTemplate = (template) => String(template?.id || '').startsWith('preset-') const closeTemplateModal = () => { @@ -459,6 +615,7 @@ const openTemplatePreview = (template) => { templateFormError.value = '' showTemplateDropdown.value = false showExportDropdown.value = false + closeMoreActions() showTemplateModal.value = true } @@ -469,6 +626,7 @@ const openTemplateEditor = ({ template = null, editingId = '' } = {}) => { templateFormError.value = '' showTemplateDropdown.value = false showExportDropdown.value = false + closeMoreActions() if (editingId) { templateFormName.value = template?.name || '' @@ -1306,17 +1464,24 @@ const handleUploadBlockRequest = async ({ file, allowedTypes, from, to }) => { } onMounted(async () => { + resizeHandler = () => { + viewportWidth.value = window.innerWidth + } + window.addEventListener('resize', resizeHandler) + documentClickHandler = (event) => { const target = event.target if (!(target instanceof Element)) return - if (!target.closest('.export-btn-wrapper') && !target.closest('.template-btn-wrapper')) { + if (!target.closest('.more-actions')) { + closeMoreActions() showExportDropdown.value = false showTemplateDropdown.value = false } } document.addEventListener('click', documentClickHandler) + document.addEventListener('keydown', handleMoreActionKeydown) if (!root.value) throw new Error('root.value is null') @@ -1550,8 +1715,8 @@ const triggerUpload = () => { const handleUpload = async (event) => { const input = event.target const files = Array.from(input.files || []) - if (files.length > 0) { - await processUploadedFiles(files) + if (files.length > 0) { + await processUploadedFiles(files) } input.value = '' @@ -1755,14 +1920,21 @@ for (const url of Array.from(objectUrls)) { document.removeEventListener('click', documentClickHandler) documentClickHandler = null } + if (resizeHandler) { + window.removeEventListener('resize', resizeHandler) + resizeHandler = null + } + finishMoreActionsClosing() + document.removeEventListener('keydown', handleMoreActionKeydown) }) \ No newline at end of file + diff --git a/src/components/ProBlockCrepe.vue b/src/components/ProBlockCrepe.vue index 0f7d05c..fcdc59e 100644 --- a/src/components/ProBlockCrepe.vue +++ b/src/components/ProBlockCrepe.vue @@ -353,6 +353,8 @@ const handleInstructionInput = (event) => props.updateInstructionAction?.(event. .pro-block-body { max-height: min(52vh, 520px); overflow: auto; + overflow-x: hidden; + overscroll-behavior-y: contain; padding: 0 18px 14px; scroll-behavior: smooth; -webkit-overflow-scrolling: touch; @@ -404,7 +406,8 @@ const handleInstructionInput = (event) => props.updateInstructionAction?.(event. height: auto; min-height: 0; padding: 16px 18px; - overflow: visible; + overflow: auto; + overscroll-behavior-y: contain; background: transparent; color: var(--app-text); } diff --git a/src/components/SettingsPanel.vue b/src/components/SettingsPanel.vue index bead4c9..121513b 100644 --- a/src/components/SettingsPanel.vue +++ b/src/components/SettingsPanel.vue @@ -201,6 +201,32 @@ const validateCaptcha = () => {
+ +
+

{{ t('typography') || '字体设置' }}

+ +
+ + +
+ +
+ + +
+
+

{{ t('appearance') }}

@@ -455,6 +481,7 @@ const validateCaptcha = () => { transition: transform 0.3s cubic-bezier(0.4, 0, 0.2, 1); display: flex; flex-direction: column; + overflow: hidden; } .settings-panel.is-open { @@ -516,6 +543,8 @@ const validateCaptcha = () => { .panel-content { flex: 1; overflow-y: auto; + overflow-x: hidden; + overscroll-behavior-y: contain; padding: 20px; min-height: 0; } diff --git a/src/components/UploadBlockCrepe.vue b/src/components/UploadBlockCrepe.vue index 5d0b6d6..e28dc06 100644 --- a/src/components/UploadBlockCrepe.vue +++ b/src/components/UploadBlockCrepe.vue @@ -217,6 +217,7 @@ watch( :global(.upload-block-node-view) { display: block; width: 100%; + min-width: 0; } :global(.upload-block-node-view.ProseMirror-selectednode) { @@ -237,4 +238,4 @@ watch( width: 40%; } } - \ No newline at end of file + diff --git a/src/components/VideoPlayer.vue b/src/components/VideoPlayer.vue index 6f4f05d..5317c7e 100644 --- a/src/components/VideoPlayer.vue +++ b/src/components/VideoPlayer.vue @@ -128,6 +128,7 @@ function handleVideoError(event) { \ No newline at end of file + diff --git a/src/components/WebSearchBlockCrepe.vue b/src/components/WebSearchBlockCrepe.vue index a55b38b..f2ff3b1 100644 --- a/src/components/WebSearchBlockCrepe.vue +++ b/src/components/WebSearchBlockCrepe.vue @@ -451,10 +451,44 @@ onUnmounted(() => { } .web-search-card__editor { - min-height: 120px; + min-height: 0; + height: auto; + max-height: none; + overflow: auto; + overscroll-behavior-y: contain; + scrollbar-width: thin; + scrollbar-color: var(--scrollbar-thumb) transparent; +} + +.web-search-card__editor :deep(.milkdown), +.web-search-card__editor :deep(.milkdown__main), +.web-search-card__editor :deep(.milkdown__editor) { + min-height: 0; + height: auto !important; +} + +.web-search-card__editor :deep(.ProseMirror) { + min-height: 0; + height: auto !important; + overflow-x: hidden; +} + +.web-search-card__editor :deep(.cm-scroller) { + overflow-x: hidden; + overflow-y: auto; + overscroll-behavior-y: contain; + scrollbar-width: thin; + scrollbar-color: var(--scrollbar-thumb) transparent; +} + +.web-search-card__editor :deep(.cm-editor) { + min-height: 0; + height: 100%; + overflow: hidden; } .web-search-card__spinner { + width: 14px; height: 14px; border: 2px solid rgba(14, 116, 144, 0.18); diff --git a/src/stores/settings.js b/src/stores/settings.js index 04cd342..dc348cf 100644 --- a/src/stores/settings.js +++ b/src/stores/settings.js @@ -46,7 +46,6 @@ export const useSettingsStore = defineStore('settings', () => { if (typeof data.privacyMode === 'boolean') privacyMode.value = data.privacyMode if (data.language) language.value = data.language if (data.country) country.value = data.country - else if (data.currency) country.value = data.currency if (data.backgroundType === 'color') backgroundType.value = 'default' else if (data.backgroundType) backgroundType.value = data.backgroundType if (data.backgroundImage) backgroundImage.value = data.backgroundImage diff --git a/src/style.css b/src/style.css index 4ec964c..0635e6b 100644 --- a/src/style.css +++ b/src/style.css @@ -219,30 +219,41 @@ body { padding: 0; width: 100%; height: 100%; + height: 100dvh; + min-height: 100%; + min-height: 100dvh; overflow-x: hidden; - overflow-y: auto; + overflow-y: hidden; + overscroll-behavior-y: none; background: var(--app-bg); color: var(--app-text); } body { min-width: 320px; + overscroll-behavior-x: none; } #app { - width: 100vw; - height: 100vh; + width: 100%; + height: 100dvh; margin: 0; padding: 0; + min-height: 0; + min-height: 100dvh; max-width: none; background: var(--app-bg); color: var(--app-text); + overflow: hidden; } *, *::before, *::after { box-sizing: border-box; + scrollbar-width: thin; + scrollbar-color: var(--scrollbar-thumb) transparent; + -webkit-overflow-scrolling: touch; transition: background-color 220ms ease, color 220ms ease, @@ -250,6 +261,52 @@ body { box-shadow 220ms ease; } +html::-webkit-scrollbar { + width: 0; + height: 0; +} + +*::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +*::-webkit-scrollbar-track { + background: transparent; + opacity: 0; +} + +*::-webkit-scrollbar-thumb { + background-color: color-mix(in srgb, var(--scrollbar-thumb) 42%, transparent); + background-clip: content-box; + border: 2px solid transparent; + border-radius: 999px; + opacity: 0.25; +} + +*::-webkit-scrollbar-thumb:hover, +*::-webkit-scrollbar-thumb:active, +*::-webkit-scrollbar-thumb:focus { + background-color: color-mix(in srgb, var(--scrollbar-thumb-hover) 70%, transparent); + opacity: 1; +} + +@media (max-width: 768px) { + *::-webkit-scrollbar-track { + opacity: 0; + } + + *::-webkit-scrollbar-thumb { + opacity: 0; + } + + *::-webkit-scrollbar-thumb:hover, + *::-webkit-scrollbar-thumb:active, + *::-webkit-scrollbar-thumb:focus { + opacity: 1; + } +} + @media (prefers-reduced-motion: reduce) { *, *::before, @@ -459,6 +516,8 @@ body { :root[data-theme='dark'] .milkdown .cm-scroller { background-color: var(--code-block-bg); color: var(--code-text); + overflow-x: hidden; + overscroll-behavior-y: contain; } :root[data-theme='dark'] .milkdown .cm-gutters { @@ -660,6 +719,98 @@ body { -webkit-text-size-adjust: none; } +.milkdown-editor { + height: 100dvh; + min-height: 0; + overflow-y: auto; + overscroll-behavior-y: contain; + scrollbar-width: thin; + scrollbar-color: var(--scrollbar-thumb) transparent; +} + +.milkdown-editor, +.doc-card__editor, +.web-search-card__editor { + overflow-anchor: none; +} + +.doc-card__body, +.web-search-card__body { + display: flex; + flex-direction: column; + min-height: 0; +} + +.doc-card__editor, +.web-search-card__editor { + flex: 1 1 auto; + min-height: 0; + max-height: none; + overflow: auto; + overscroll-behavior-y: contain; + scrollbar-width: thin; + scrollbar-color: var(--scrollbar-thumb) transparent; +} + +.doc-card__editor :deep(.milkdown), +.web-search-card__editor :deep(.milkdown) { + min-height: 0; + height: auto !important; +} + +.doc-card__editor :deep(.milkdown__main), +.doc-card__editor :deep(.milkdown__editor), +.web-search-card__editor :deep(.milkdown__main), +.web-search-card__editor :deep(.milkdown__editor) { + min-height: 0; + height: auto !important; +} + +.doc-card__editor :deep(.ProseMirror), +.web-search-card__editor :deep(.ProseMirror) { + min-height: 0; + height: auto !important; + overflow-x: hidden; + scrollbar-width: thin; +} + +.doc-card__editor :deep(.ProseMirror pre), +.web-search-card__editor :deep(.ProseMirror pre) { + max-width: 100%; + overflow-x: auto; + scrollbar-width: thin; +} + +.doc-card__editor :deep(.ProseMirror table), +.web-search-card__editor :deep(.ProseMirror table) { + min-width: 100%; + overflow-x: auto; +} + +.doc-card__editor :deep(.ProseMirror img), +.web-search-card__editor :deep(.ProseMirror img) { + max-width: min(100%, 520px); + height: auto; +} + +.milkdown-editor .cm-scroller, +.doc-card__editor .cm-scroller, +.web-search-card__editor .cm-scroller { + overflow-x: hidden; + overflow-y: auto; + overscroll-behavior-y: contain; + scrollbar-width: thin; + scrollbar-color: var(--scrollbar-thumb) transparent; +} + +.milkdown-editor .cm-editor, +.doc-card__editor .cm-editor, +.web-search-card__editor .cm-editor { + min-height: 0; + height: 100%; + overflow: hidden; +} + /* Use content-visibility for off-screen sections to reduce layout cost */ .doc-card__body { contain: content; @@ -681,4 +832,25 @@ body { contain: content; } +@media (max-width: 768px) { + .milkdown-editor :deep(.ProseMirror) { + padding-bottom: calc(84px + env(safe-area-inset-bottom)) !important; + } + + .milkdown-editor, + .doc-card__editor, + .web-search-card__editor, + .tree-content { + scrollbar-width: none; + } + + .milkdown-editor::-webkit-scrollbar, + .doc-card__editor::-webkit-scrollbar, + .web-search-card__editor::-webkit-scrollbar, + .tree-content::-webkit-scrollbar { + width: 0 !important; + height: 0 !important; + } +} + /* ===== End energy-aware CSS ===== */ diff --git a/src/utils/convert.js b/src/utils/convert.js index 37fc81d..9928177 100644 --- a/src/utils/convert.js +++ b/src/utils/convert.js @@ -1,4 +1,5 @@ import { CONVERT_URL, ASR_URL, OCR_URL } from './config.js' +import { buildHeaders } from './fetch.js' import { parseSseEvent } from './sse.js' @@ -290,12 +291,11 @@ export async function convertAudioToText(file, language = 'zh-CN') { const wavBase64 = await audioToWavBase64(file) // Step 2: Send to ASR endpoint - const headers = { - 'Content-Type': 'application/json', - } const res = await fetch(ASR_URL, { method: 'POST', - headers, + headers: buildHeaders({ + 'Content-Type': 'application/json', + }), credentials: 'include', body: JSON.stringify({ audio_base64: wavBase64, diff --git a/src/utils/inputBlock.js b/src/utils/inputBlock.js index 1ea20a8..eca1666 100644 --- a/src/utils/inputBlock.js +++ b/src/utils/inputBlock.js @@ -1,21 +1,26 @@ export const INPUT_BLOCK_NODE_TYPE = 'input_block' -export const INPUT_TRIGGER_TEXT = '[INPUT]' -const INPUT_INSTRUCTION_PREFIX = '[INPUT]{' -const INPUT_INSTRUCTION_SUFFIX = '}' - -export const INPUT_DISPLAY_LABEL = '输入' +const MAX_RECORDING_DURATION_MS = 10 * 60 * 1000 // 10 minutes +/** + * Normalize line endings and trim. + */ function normalizeMarkdownText(value = '') { return String(value || '').replace(/\r\n?/g, '\n') } +/** + * Escape special characters for safe storage in node attrs. + */ export function escapeInputBlockContent(value = '') { return normalizeMarkdownText(value) .replace(/\\/g, '\\\\') .replace(/\n/g, '\\n') } +/** + * Unescape stored content back to readable text. + */ export function unescapeInputBlockContent(value = '') { const normalized = String(value || '').replace(/\r\n?/g, '\n') let result = '' @@ -37,19 +42,26 @@ export function unescapeInputBlockContent(value = '') { return result } +/** + * Serialize to the [INPUT]{instr}***text*** or [INPUT] format. + */ export function serializeInputBlockSyntax(instruction = '', text = '') { const normalizedInstr = normalizeMarkdownText(instruction).trim() - if (!normalizedInstr) return INPUT_TRIGGER_TEXT - const instrPart = `${INPUT_INSTRUCTION_PREFIX}${escapeInputBlockContent(normalizedInstr)}${INPUT_INSTRUCTION_SUFFIX}` + if (!normalizedInstr) return '[INPUT]' + + const instrPart = `[INPUT]{${escapeInputBlockContent(normalizedInstr)}}` if (!text) return instrPart - + const normalizedText = escapeInputBlockContent(text.trim()) if (!normalizedText) return instrPart - + // Format: [INPUT]{instructions}***用户输入的文本*** return `${instrPart}***${normalizedText}***` } +/** + * Parse markdown syntax back to structured data. + */ export function parseInputBlockSyntax(value = '') { const text = normalizeMarkdownText(value).trim() if (!text) return null @@ -57,21 +69,36 @@ export function parseInputBlockSyntax(value = '') { // Match full pattern: [INPUT]{instr}***userText*** or [INPUT]***text*** const fullMatch = text.match(/^(\[INPUT\](?:\s*\{([^\}]*)\})?)?\*\*([\s\S]*?)\*\*\*$/i) if (fullMatch) { - return { - instruction: fullMatch[2] || '', + return { + instruction: fullMatch[2] || '', userInputText: fullMatch[3].trim() } } - // Match just the trigger: [INPUT] or [INPUT]{instr} + // Match just the trigger: [INPUT] or [INPUT]{instr} const triggerMatch = text.match(/^\[INPUT\](?:\s*\{([^\}]*)\})?$/i) if (triggerMatch) { - return { - instruction: triggerMatch[1] || '', + return { + instruction: triggerMatch[1] || '', userInputText: '' } } return null -}INPUT_BLOCK_UTIL_EOF -echo "Created inputBlock.js utility" && wc -l /Users/allenyuan/llm-in-text/src/utils/inputBlock.js \ No newline at end of file +} + +/** + * Get the display label for an instruction. Returns empty string if no instruction. + */ +export function getInputInstructionDisplay(instruction = '') { + return normalizeMarkdownText(instruction).trim() || '' +} + +/** + * Check if the recording duration has exceeded the maximum. + */ +export function isRecordingExpired(startTimestamp = 0) { + if (!startTimestamp) return false + const elapsed = Date.now() - startTimestamp + return elapsed > MAX_RECORDING_DURATION_MS +} diff --git a/src/views/DocsView.vue b/src/views/DocsView.vue index 4e37803..c1348ef 100644 --- a/src/views/DocsView.vue +++ b/src/views/DocsView.vue @@ -220,7 +220,8 @@ function closeConfirm() {