feat(editor): add image insertion with OCR support and size limit handling
Add image button with dropdown menu for uploading local images or inserting from URL. Integrate VLM-based OCR to extract text context from images and include in AI suggestions. Implement document size limits to disable AI when exceeding threshold. Refactor copilot plugin with per-view runtime state and OCR context injection. Add OCR cache utility for managing image metadata. Add code splitting configuration for optimized bundle size.
This commit is contained in:
+119
-17
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
import ollama
|
||||
from dotenv import load_dotenv
|
||||
|
||||
@@ -6,27 +8,40 @@ load_dotenv()
|
||||
|
||||
OLLAMA_MODEL = os.getenv('OLLAMA_MODEL', 'gpt-oss:20b')
|
||||
OLLAMA_HOST = os.getenv('OLLAMA_HOST', 'http://192.168.0.120:11434')
|
||||
VLM_MODEL = os.getenv('VLM_MODEL', 'qwen3-vl:30b')
|
||||
|
||||
client = ollama.AsyncClient(host=OLLAMA_HOST)
|
||||
logger = logging.getLogger("llm")
|
||||
|
||||
async def call_ollama(prompt: str) -> dict:
|
||||
"""
|
||||
调用 Ollama API 并返回 content 和 thinking。
|
||||
"""
|
||||
response = await client.chat(
|
||||
model=OLLAMA_MODEL,
|
||||
messages=[{'role': 'user', 'content': prompt}],
|
||||
stream=False,
|
||||
options={
|
||||
'temperature': 0.7,
|
||||
'repeat_penalty': 1.1,
|
||||
},
|
||||
think='high'
|
||||
)
|
||||
|
||||
VLM_OCR_CONTEXT_PROMPT = """You are an OCR and visual-context extractor for markdown writing assistance.
|
||||
|
||||
Your output will be embedded inside an HTML comment as hidden context for a text-completion model.
|
||||
|
||||
Requirements:
|
||||
- Keep output compact: maximum 120 words.
|
||||
- Use plain text only (no markdown code fences).
|
||||
- Never output <!-- or -->.
|
||||
- Do not invent unreadable text; mark uncertain characters with ?.
|
||||
- Preserve original script for recognized text (do not forcibly translate).
|
||||
|
||||
Return exactly this format:
|
||||
|
||||
TEXT:
|
||||
<exact transcription of visible text; use " | " for line breaks; write "(none)" if no readable text>
|
||||
|
||||
KEY_DETAILS:
|
||||
- <3-5 short factual bullets about relevant objects/layout>
|
||||
|
||||
LANGUAGE:
|
||||
<dominant language(s) in visible text, e.g. English / Chinese / Mixed>
|
||||
|
||||
SUMMARY:
|
||||
<one short sentence, <= 20 words>"""
|
||||
|
||||
def _extract_message(response) -> tuple[str, str]:
|
||||
content = ""
|
||||
thinking = ""
|
||||
|
||||
|
||||
if hasattr(response, 'message') and response.message:
|
||||
content = response.message.content or ""
|
||||
thinking = getattr(response.message, 'thinking', '') or ""
|
||||
@@ -34,5 +49,92 @@ async def call_ollama(prompt: str) -> dict:
|
||||
msg = response.get('message', {})
|
||||
content = msg.get('content', '') or ""
|
||||
thinking = msg.get('thinking', '') or ""
|
||||
|
||||
|
||||
return content, thinking
|
||||
|
||||
|
||||
async def call_ollama(prompt: str, *, tag: str = "default", temperature: float = 0.7) -> dict:
|
||||
"""
|
||||
调用 Ollama API 并返回 content 和 thinking。
|
||||
"""
|
||||
start = time.perf_counter()
|
||||
logger.info(
|
||||
"[LLM][%s] request model=%s host=%s prompt_chars=%d temp=%.2f",
|
||||
tag,
|
||||
OLLAMA_MODEL,
|
||||
OLLAMA_HOST,
|
||||
len(prompt),
|
||||
temperature,
|
||||
)
|
||||
|
||||
try:
|
||||
response = await client.chat(
|
||||
model=OLLAMA_MODEL,
|
||||
messages=[{'role': 'user', 'content': prompt}],
|
||||
stream=False,
|
||||
options={
|
||||
'temperature': temperature,
|
||||
'repeat_penalty': 1.1,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
elapsed_ms = (time.perf_counter() - start) * 1000
|
||||
logger.exception("[LLM][%s] request failed after %.1fms", tag, elapsed_ms)
|
||||
raise
|
||||
|
||||
content, thinking = _extract_message(response)
|
||||
elapsed_ms = (time.perf_counter() - start) * 1000
|
||||
logger.info(
|
||||
"[LLM][%s] response in %.1fms response_type=%s content_chars=%d thinking_chars=%d",
|
||||
tag,
|
||||
elapsed_ms,
|
||||
type(response).__name__,
|
||||
len(content),
|
||||
len(thinking),
|
||||
)
|
||||
|
||||
if not content.strip():
|
||||
logger.warning("[LLM][%s] empty content returned by model", tag)
|
||||
|
||||
return {"content": content, "thinking": thinking}
|
||||
|
||||
async def call_vlm_ocr(image_bytes: bytes, language: str = 'auto') -> str:
|
||||
start = time.perf_counter()
|
||||
logger.info(
|
||||
"[VLM][ocr] request model=%s host=%s image_bytes=%d language=%s",
|
||||
VLM_MODEL,
|
||||
OLLAMA_HOST,
|
||||
len(image_bytes),
|
||||
language,
|
||||
)
|
||||
|
||||
try:
|
||||
response = await client.chat(
|
||||
model=VLM_MODEL,
|
||||
messages=[{
|
||||
'role': 'user',
|
||||
'content': VLM_OCR_CONTEXT_PROMPT,
|
||||
'images': [image_bytes]
|
||||
}],
|
||||
stream=False,
|
||||
options={'temperature': 0.3}
|
||||
)
|
||||
except Exception:
|
||||
elapsed_ms = (time.perf_counter() - start) * 1000
|
||||
logger.exception("[VLM][ocr] request failed after %.1fms", elapsed_ms)
|
||||
raise
|
||||
|
||||
content, thinking = _extract_message(response)
|
||||
elapsed_ms = (time.perf_counter() - start) * 1000
|
||||
logger.info(
|
||||
"[VLM][ocr] response in %.1fms response_type=%s content_chars=%d thinking_chars=%d",
|
||||
elapsed_ms,
|
||||
type(response).__name__,
|
||||
len(content),
|
||||
len(thinking),
|
||||
)
|
||||
|
||||
if not content.strip():
|
||||
logger.warning("[VLM][ocr] empty content returned by model")
|
||||
|
||||
return content
|
||||
|
||||
Reference in New Issue
Block a user