feat: add video OCR/ASR, DOCX/PDF export, input block and risk config updates

- Video pipeline: video file OCR via VLM plus audio track ASR, integrated
  into job_handlers with progress emit per phase. New media_utils.py for
  audio extraction from video files.
- Document export: richExport.js replaces inline docx builder; DOCX and PDF
  export buttons are now enabled in MilkdownEditor. File size limit raised to
  100 MB.
- Input block: new InputBlockCrepe.vue component with inputBlockPlugin.ts
  and inputBlock.js for custom user-input nodes in the editor.
- Risk config: added Vite dev server ports (5173) to CORS allowlist and
  increased OCR max input from 10 MB to 100 MB.
- TTS/ASR refactor: simplified tts_asr.py model loading and warmup logic.
- Test coverage: updated tests for llm, main endpoints, pro completions and
  web search modules.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
“ydy0615”
2026-06-18 16:32:31 +08:00
parent 4813196b0a
commit 356108e792
34 changed files with 1457 additions and 563 deletions
+52 -5
View File
@@ -14,6 +14,7 @@ import markitdown
from audit_store import get_audit_store
from llm import call_ollama, call_vlm_ocr, stream_ollama_events
from media_utils import extract_audio_wav_bytes, is_video_filename
from prompt import (
build_completion_prompts,
build_pro_completion_prompts,
@@ -720,14 +721,60 @@ async def ocr_handler(
path = payload["input_path"]
identity, risk, lock_keys = await _enter_llm_execution(payload, emit)
try:
filename = payload.get("filename", "image.jpg")
language = payload.get("language", "auto")
media_type = payload.get("media_type", "image")
mime_type = payload.get("mime_type", "") or ""
with open(path, "rb") as handle:
image_bytes = handle.read()
text = await call_vlm_ocr(image_bytes, payload.get("language", "auto"))
media_bytes = handle.read()
await emit("progress", {"phase": "ocr", "media_type": media_type})
ocr_text = await call_vlm_ocr(
media_bytes,
language,
mime_type=mime_type or "application/octet-stream",
media_type=media_type,
)
if is_cancelled():
raise asyncio.CancelledError()
await emit("result", {"text": text})
await _exit_llm_execution(payload, identity, risk, lock_keys, status="completed", actual_output_text=text)
return {"text": text, "filename": payload.get("filename", "image.jpg")}
result = {
"text": ocr_text,
"ocr_text": ocr_text,
"filename": filename,
"media_type": media_type,
}
if media_type == "video" or is_video_filename(filename, mime_type):
asr_text = ""
if generate_asr_response is not None:
try:
await emit("progress", {"phase": "asr", "media_type": media_type})
audio_bytes = await asyncio.to_thread(extract_audio_wav_bytes, path)
asr_response = await generate_asr_response(audio_bytes, language)
asr_text = getattr(asr_response, "text", "") or ""
except Exception as exc:
asr_text = f"(音频解析失败: {exc})"
if ocr_text.strip() or asr_text.strip():
text_parts = []
if ocr_text.strip():
text_parts.append(f"## 视频画面 OCR\n\n{ocr_text.strip()}")
if asr_text.strip():
text_parts.append(f"## 视频音频 ASR\n\n{asr_text.strip()}")
result["text"] = "\n\n".join(text_parts)
result["asr_text"] = asr_text
await emit("result", result)
await _exit_llm_execution(
payload,
identity,
risk,
lock_keys,
status="completed",
actual_output_text=result["text"],
)
return result
except asyncio.CancelledError:
await _exit_llm_execution(payload, identity, risk, lock_keys, status="cancelled", error_code="cancelled")
raise