feat: add video OCR/ASR, DOCX/PDF export, input block and risk config updates
- Video pipeline: video file OCR via VLM plus audio track ASR, integrated into job_handlers with progress emit per phase. New media_utils.py for audio extraction from video files. - Document export: richExport.js replaces inline docx builder; DOCX and PDF export buttons are now enabled in MilkdownEditor. File size limit raised to 100 MB. - Input block: new InputBlockCrepe.vue component with inputBlockPlugin.ts and inputBlock.js for custom user-input nodes in the editor. - Risk config: added Vite dev server ports (5173) to CORS allowlist and increased OCR max input from 10 MB to 100 MB. - TTS/ASR refactor: simplified tts_asr.py model loading and warmup logic. - Test coverage: updated tests for llm, main endpoints, pro completions and web search modules. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
+52
-5
@@ -14,6 +14,7 @@ import markitdown
|
||||
|
||||
from audit_store import get_audit_store
|
||||
from llm import call_ollama, call_vlm_ocr, stream_ollama_events
|
||||
from media_utils import extract_audio_wav_bytes, is_video_filename
|
||||
from prompt import (
|
||||
build_completion_prompts,
|
||||
build_pro_completion_prompts,
|
||||
@@ -720,14 +721,60 @@ async def ocr_handler(
|
||||
path = payload["input_path"]
|
||||
identity, risk, lock_keys = await _enter_llm_execution(payload, emit)
|
||||
try:
|
||||
filename = payload.get("filename", "image.jpg")
|
||||
language = payload.get("language", "auto")
|
||||
media_type = payload.get("media_type", "image")
|
||||
mime_type = payload.get("mime_type", "") or ""
|
||||
|
||||
with open(path, "rb") as handle:
|
||||
image_bytes = handle.read()
|
||||
text = await call_vlm_ocr(image_bytes, payload.get("language", "auto"))
|
||||
media_bytes = handle.read()
|
||||
|
||||
await emit("progress", {"phase": "ocr", "media_type": media_type})
|
||||
ocr_text = await call_vlm_ocr(
|
||||
media_bytes,
|
||||
language,
|
||||
mime_type=mime_type or "application/octet-stream",
|
||||
media_type=media_type,
|
||||
)
|
||||
if is_cancelled():
|
||||
raise asyncio.CancelledError()
|
||||
await emit("result", {"text": text})
|
||||
await _exit_llm_execution(payload, identity, risk, lock_keys, status="completed", actual_output_text=text)
|
||||
return {"text": text, "filename": payload.get("filename", "image.jpg")}
|
||||
|
||||
result = {
|
||||
"text": ocr_text,
|
||||
"ocr_text": ocr_text,
|
||||
"filename": filename,
|
||||
"media_type": media_type,
|
||||
}
|
||||
|
||||
if media_type == "video" or is_video_filename(filename, mime_type):
|
||||
asr_text = ""
|
||||
if generate_asr_response is not None:
|
||||
try:
|
||||
await emit("progress", {"phase": "asr", "media_type": media_type})
|
||||
audio_bytes = await asyncio.to_thread(extract_audio_wav_bytes, path)
|
||||
asr_response = await generate_asr_response(audio_bytes, language)
|
||||
asr_text = getattr(asr_response, "text", "") or ""
|
||||
except Exception as exc:
|
||||
asr_text = f"(音频解析失败: {exc})"
|
||||
if ocr_text.strip() or asr_text.strip():
|
||||
text_parts = []
|
||||
if ocr_text.strip():
|
||||
text_parts.append(f"## 视频画面 OCR\n\n{ocr_text.strip()}")
|
||||
if asr_text.strip():
|
||||
text_parts.append(f"## 视频音频 ASR\n\n{asr_text.strip()}")
|
||||
result["text"] = "\n\n".join(text_parts)
|
||||
result["asr_text"] = asr_text
|
||||
|
||||
await emit("result", result)
|
||||
await _exit_llm_execution(
|
||||
payload,
|
||||
identity,
|
||||
risk,
|
||||
lock_keys,
|
||||
status="completed",
|
||||
actual_output_text=result["text"],
|
||||
)
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
await _exit_llm_execution(payload, identity, risk, lock_keys, status="cancelled", error_code="cancelled")
|
||||
raise
|
||||
|
||||
Reference in New Issue
Block a user