feat: add video OCR/ASR, DOCX/PDF export, input block and risk config updates

- Video pipeline: video file OCR via VLM plus audio track ASR, integrated
  into job_handlers with progress emit per phase. New media_utils.py for
  audio extraction from video files.
- Document export: richExport.js replaces inline docx builder; DOCX and PDF
  export buttons are now enabled in MilkdownEditor. File size limit raised to
  100 MB.
- Input block: new InputBlockCrepe.vue component with inputBlockPlugin.ts
  and inputBlock.js for custom user-input nodes in the editor.
- Risk config: added Vite dev server ports (5173) to CORS allowlist and
  increased OCR max input from 10 MB to 100 MB.
- TTS/ASR refactor: simplified tts_asr.py model loading and warmup logic.
- Test coverage: updated tests for llm, main endpoints, pro completions and
  web search modules.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
“ydy0615”
2026-06-18 16:32:31 +08:00
parent 4813196b0a
commit 356108e792
34 changed files with 1457 additions and 563 deletions
+1
View File
@@ -299,3 +299,4 @@ def test_call_vlm_ocr(monkeypatch):
image_part = [p for p in content_parts if p.get("type") == "image_url"]
assert len(image_part) == 1
assert image_part[0]["image_url"]["url"].startswith("data:image/png;base64,")
assert captured["json"]["options"]["think"] is False
+30
View File
@@ -4,6 +4,7 @@ import importlib
import os
import sys
from pathlib import Path
from types import SimpleNamespace
from fastapi.testclient import TestClient
@@ -158,6 +159,35 @@ def test_post_ocr_mocked(monkeypatch):
assert "OCR result text" in body
def test_post_video_ocr_merges_ocr_and_asr(monkeypatch):
async def fake_ocr(*args, **kwargs):
return "画面文字"
async def fake_asr(*args, **kwargs):
return SimpleNamespace(text="音频转写")
monkeypatch.setattr(job_handlers, "call_vlm_ocr", fake_ocr)
monkeypatch.setattr(job_handlers, "generate_asr_response", fake_asr)
monkeypatch.setattr(job_handlers, "extract_audio_wav_bytes", lambda _path: b"fake wav")
video_b64 = base64.b64encode(b"pretend video data").decode()
with TestClient(main.app) as client:
with client.stream("POST", "/v1/ocr", headers=HEADERS, json={
"image": video_b64,
"filename": "sample.mp4",
"language": "auto",
"media_type": "video",
"mime_type": "video/mp4",
}) as resp:
assert resp.status_code == 200
body = "".join(resp.iter_text())
assert "视频画面 OCR" in body
assert "视频音频 ASR" in body
assert "画面文字" in body
assert "音频转写" in body
def test_post_convert_txt_returns_markdown():
content = base64.b64encode(b"hello world").decode()
with TestClient(main.app) as client:
+4 -4
View File
@@ -35,7 +35,7 @@ def _payload():
"privacy_mode": True,
"user_preferences": {
"language": "zh",
"currency": "CNY",
"country": "CN",
"timezone": "Asia/Shanghai",
},
}
@@ -43,7 +43,7 @@ def _payload():
def test_pro_queue_full_returns_429(monkeypatch):
async def fake_queue_job(*args, **kwargs):
raise job_system.QueueFullError("pro_completion queue is full")
raise job_system.QueueFullError("pro_completion", 8)
monkeypatch.setattr(main, "_queue_job", fake_queue_job)
with TestClient(main.app) as client:
@@ -79,13 +79,13 @@ def test_pro_prompt_accepts_serialized_preferences():
instruction="expand",
preferences={
"language": "zh",
"currency": "CNY",
"country": "CN",
"timezone": "Asia/Shanghai",
},
)
assert "Preferred language: zh" in user_prompt
assert "Preferred currency: CNY" in user_prompt
assert "Preferred country: CN" in user_prompt
assert "Preferred timezone: Asia/Shanghai" in user_prompt
+8 -2
View File
@@ -17,6 +17,7 @@ if str(BACKEND_DIR) not in sys.path:
import job_handlers # type: ignore
import job_system # type: ignore
import llm # type: ignore
import risk_control # type: ignore
import session_store # type: ignore
import audit_store # type: ignore
@@ -55,10 +56,14 @@ def test_web_search_route_returns_done(monkeypatch):
return {"content": '["vector database comparison", "pinecone weaviate qdrant"]'}
if tag.endswith("-webu"):
return {"content": '["https://example.com/a", "https://example.com/b"]'}
if tag.endswith("-webf"):
return {"content": "第一段\n\n第二段"}
raise AssertionError(f"unexpected tag: {tag}")
async def fake_stream_ollama_events(prompt, system_prompt=None, tag="", **kwargs): # noqa: ARG001
if not tag.endswith("-webf"):
raise AssertionError(f"unexpected stream tag: {tag}")
yield "content", "第一段\n\n"
yield "content", "第二段"
async def fake_searxng_search(query, *, limit): # noqa: ARG001
return [
{
@@ -83,6 +88,7 @@ def test_web_search_route_returns_done(monkeypatch):
monkeypatch.setattr(job_handlers, "call_ollama", fake_call_ollama)
monkeypatch.setattr(job_handlers, "_searxng_search", fake_searxng_search)
monkeypatch.setattr(job_handlers, "_firecrawl_scrape", fake_firecrawl_scrape)
monkeypatch.setattr(llm, "stream_ollama_events", fake_stream_ollama_events)
with TestClient(main.app) as client:
with client.stream("POST", "/v1/web-search", headers=HEADERS, json=_payload()) as resp: