feat(editor): add image insertion with OCR support and size limit handling

Add image button with dropdown menu for uploading local images or inserting from URL.
Integrate VLM-based OCR to extract text context from images and include in AI suggestions.
Implement document size limits to disable AI when exceeding threshold.
Refactor copilot plugin with per-view runtime state and OCR context injection.
Add OCR cache utility for managing image metadata.
Add code splitting configuration for optimized bundle size.
This commit is contained in:
“ydy0615”
2026-02-14 18:28:37 +08:00
parent c64ff7be45
commit 64cfa58376
16 changed files with 1593 additions and 458 deletions
+1
View File
@@ -1,3 +1,4 @@
OPENAI_API_KEY=ollama
OLLAMA_HOST=http://192.168.0.120:11434
OLLAMA_MODEL=gpt-oss:20b
VLM_MODEL=qwen3-vl:30b
+1
View File
@@ -1,3 +1,4 @@
OPENAI_API_KEY=ollama
OLLAMA_BASE_URL=http://192.168.0.120:11434/v1/
OLLAMA_MODEL=gpt-oss:120b
VLM_MODEL=qwen3-vl:30b
Binary file not shown.
Binary file not shown.
Binary file not shown.
+119 -17
View File
@@ -1,4 +1,6 @@
import os
import time
import logging
import ollama
from dotenv import load_dotenv
@@ -6,27 +8,40 @@ load_dotenv()
OLLAMA_MODEL = os.getenv('OLLAMA_MODEL', 'gpt-oss:20b')
OLLAMA_HOST = os.getenv('OLLAMA_HOST', 'http://192.168.0.120:11434')
VLM_MODEL = os.getenv('VLM_MODEL', 'qwen3-vl:30b')
client = ollama.AsyncClient(host=OLLAMA_HOST)
logger = logging.getLogger("llm")
async def call_ollama(prompt: str) -> dict:
"""
调用 Ollama API 并返回 content 和 thinking。
"""
response = await client.chat(
model=OLLAMA_MODEL,
messages=[{'role': 'user', 'content': prompt}],
stream=False,
options={
'temperature': 0.7,
'repeat_penalty': 1.1,
},
think='high'
)
VLM_OCR_CONTEXT_PROMPT = """You are an OCR and visual-context extractor for markdown writing assistance.
Your output will be embedded inside an HTML comment as hidden context for a text-completion model.
Requirements:
- Keep output compact: maximum 120 words.
- Use plain text only (no markdown code fences).
- Never output <!-- or -->.
- Do not invent unreadable text; mark uncertain characters with ?.
- Preserve original script for recognized text (do not forcibly translate).
Return exactly this format:
TEXT:
<exact transcription of visible text; use " | " for line breaks; write "(none)" if no readable text>
KEY_DETAILS:
- <3-5 short factual bullets about relevant objects/layout>
LANGUAGE:
<dominant language(s) in visible text, e.g. English / Chinese / Mixed>
SUMMARY:
<one short sentence, <= 20 words>"""
def _extract_message(response) -> tuple[str, str]:
content = ""
thinking = ""
if hasattr(response, 'message') and response.message:
content = response.message.content or ""
thinking = getattr(response.message, 'thinking', '') or ""
@@ -34,5 +49,92 @@ async def call_ollama(prompt: str) -> dict:
msg = response.get('message', {})
content = msg.get('content', '') or ""
thinking = msg.get('thinking', '') or ""
return content, thinking
async def call_ollama(prompt: str, *, tag: str = "default", temperature: float = 0.7) -> dict:
"""
调用 Ollama API 并返回 content 和 thinking。
"""
start = time.perf_counter()
logger.info(
"[LLM][%s] request model=%s host=%s prompt_chars=%d temp=%.2f",
tag,
OLLAMA_MODEL,
OLLAMA_HOST,
len(prompt),
temperature,
)
try:
response = await client.chat(
model=OLLAMA_MODEL,
messages=[{'role': 'user', 'content': prompt}],
stream=False,
options={
'temperature': temperature,
'repeat_penalty': 1.1,
},
)
except Exception:
elapsed_ms = (time.perf_counter() - start) * 1000
logger.exception("[LLM][%s] request failed after %.1fms", tag, elapsed_ms)
raise
content, thinking = _extract_message(response)
elapsed_ms = (time.perf_counter() - start) * 1000
logger.info(
"[LLM][%s] response in %.1fms response_type=%s content_chars=%d thinking_chars=%d",
tag,
elapsed_ms,
type(response).__name__,
len(content),
len(thinking),
)
if not content.strip():
logger.warning("[LLM][%s] empty content returned by model", tag)
return {"content": content, "thinking": thinking}
async def call_vlm_ocr(image_bytes: bytes, language: str = 'auto') -> str:
start = time.perf_counter()
logger.info(
"[VLM][ocr] request model=%s host=%s image_bytes=%d language=%s",
VLM_MODEL,
OLLAMA_HOST,
len(image_bytes),
language,
)
try:
response = await client.chat(
model=VLM_MODEL,
messages=[{
'role': 'user',
'content': VLM_OCR_CONTEXT_PROMPT,
'images': [image_bytes]
}],
stream=False,
options={'temperature': 0.3}
)
except Exception:
elapsed_ms = (time.perf_counter() - start) * 1000
logger.exception("[VLM][ocr] request failed after %.1fms", elapsed_ms)
raise
content, thinking = _extract_message(response)
elapsed_ms = (time.perf_counter() - start) * 1000
logger.info(
"[VLM][ocr] response in %.1fms response_type=%s content_chars=%d thinking_chars=%d",
elapsed_ms,
type(response).__name__,
len(content),
len(thinking),
)
if not content.strip():
logger.warning("[VLM][ocr] empty content returned by model")
return content
+97 -12
View File
@@ -3,9 +3,18 @@ from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse, JSONResponse
from pydantic import BaseModel
import json
import base64
import uuid
import logging
from prompt import build_prompt
from llm import call_ollama
from llm import call_ollama, call_vlm_ocr
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s - %(message)s",
)
logger = logging.getLogger("api")
app = FastAPI()
@@ -22,24 +31,100 @@ class CompletionRequest(BaseModel):
suffix: str
languageId: str = 'markdown'
class OCRRequest(BaseModel):
image: str
filename: str = "image.jpg"
language: str = 'auto'
def _preview(text: str, limit: int = 80) -> str:
value = (text or "").replace("\n", "\\n")
if len(value) <= limit:
return value
return value[:limit] + "..."
def _build_force_non_empty_prompt(base_prompt: str) -> str:
return (
base_prompt
+ "\n\nStrict override for this request:\n"
+ "- Output must be non-empty.\n"
+ "- If you would otherwise output empty, output a single space.\n"
+ "- Keep it short and do not repeat SUFFIX.\n"
)
@app.post("/v1/completions")
async def create_completion(request: CompletionRequest):
request_id = str(uuid.uuid4())[:8]
try:
prompt = build_prompt(request.prefix, request.suffix)
result = await call_ollama(prompt)
content = result["content"]
logger.info(
"[%s] /v1/completions prefix_chars=%d suffix_chars=%d lang=%s prefix_tail='%s' suffix_head='%s'",
request_id,
len(request.prefix or ""),
len(request.suffix or ""),
request.languageId,
_preview((request.prefix or "")[-120:]),
_preview((request.suffix or "")[:120]),
)
prompt = build_prompt(request.prefix, request.suffix, request.languageId)
result = await call_ollama(prompt, tag=f"{request_id}-primary", temperature=0.7)
content = result["content"] or ""
source = "primary"
if not content.strip():
logger.warning("[%s] primary returned empty content, starting retry", request_id)
retry_prompt = _build_force_non_empty_prompt(prompt)
retry_result = await call_ollama(retry_prompt, tag=f"{request_id}-retry1", temperature=0.4)
content = retry_result["content"] or ""
source = "retry1"
if not content.strip():
content = " "
source = "fallback-space"
logger.warning("[%s] retry still empty, forcing single-space fallback", request_id)
logger.info(
"[%s] completion resolved source=%s content_chars=%d content_preview='%s'",
request_id,
source,
len(content),
_preview(content, 120),
)
async def generate():
if content:
yield f"data: {json.dumps({'content': content})}\n\n"
yield f"data: {json.dumps({'content': content})}\n\n"
yield f"data: {json.dumps({'done': True})}\n\n"
return StreamingResponse(generate(), media_type="text/event-stream")
except Exception as e:
import traceback
traceback.print_exc()
logger.exception("[%s] /v1/completions failed: %s", request_id, e)
return JSONResponse(content={"error": str(e)}, status_code=500)
@app.post("/v1/ocr")
async def ocr_image(request: OCRRequest):
request_id = str(uuid.uuid4())[:8]
try:
logger.info(
"[%s] /v1/ocr filename=%s language=%s image_base64_chars=%d",
request_id,
request.filename,
request.language,
len(request.image or ""),
)
image_bytes = base64.b64decode(request.image)
logger.info("[%s] /v1/ocr decoded image_bytes=%d", request_id, len(image_bytes))
result = await call_vlm_ocr(image_bytes, request.language)
logger.info(
"[%s] /v1/ocr success text_chars=%d text_preview='%s'",
request_id,
len(result or ""),
_preview(result or "", 120),
)
return {"text": result, "filename": request.filename}
except Exception as e:
logger.exception("[%s] /v1/ocr failed: %s", request_id, e)
return JSONResponse(content={"error": str(e)}, status_code=500)
if __name__ == "__main__":
+63 -181
View File
@@ -1,202 +1,84 @@
import os
from typing import Tuple
def build_prompt(prefix: str, suffix: str) -> str:
MAX_PREFIX_CHARS = 12000
MAX_SUFFIX_CHARS = 4000
def _sanitize_language_id(language_id: str) -> str:
if not language_id:
return "markdown"
allowed = []
for ch in language_id.strip():
if ch.isalnum() or ch in "-_+.":
allowed.append(ch)
value = "".join(allowed)[:32]
return value or "markdown"
def _prepare_context(prefix: str, suffix: str) -> Tuple[str, str]:
"""
优化后的提示词构建函数。
使用明确的分隔符区分指令部分和实际的 prefix/suffix 内容。
Prepare prefix/suffix for model completion context.
Keep the historical one-char lookahead behavior to reduce boundary drift.
"""
# 修正:把suffix的第一个字符移到prefix末尾(解决光标位置偏差)
if suffix:
first_char = suffix[0]
prefix = prefix + first_char
prefix = prefix + suffix[0]
suffix = suffix[1:]
recent_prefix = prefix
recent_suffix = suffix
return prefix[-MAX_PREFIX_CHARS:], suffix[:MAX_SUFFIX_CHARS]
prompt = f"""You are an expert writing assistant integrated into a text editor. Your task is to complete the text at the cursor position.
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
RULES
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
def build_prompt(prefix: str, suffix: str, language_id: str = "markdown") -> str:
safe_language_id = _sanitize_language_id(language_id)
recent_prefix, recent_suffix = _prepare_context(prefix, suffix)
RULE #1: SEAMLESS CONNECTION (MOST CRITICAL)
prompt = f"""You are an inline completion engine for a {safe_language_id} editor with ghost-text suggestions.
Your continuation MUST seamlessly bridge the prefix and suffix. This is the MOST IMPORTANT rule.
Your job:
- Return ONLY the text that should be inserted at the cursor between PREFIX and SUFFIX.
- Prefer a meaningful, non-empty insertion with moderate length.
- Avoid overly short outputs with little information value.
The "复读机" (Parrot) Error is when you repeat content that already exists in the suffix. This is the WORST mistake you can make.
Important context:
- PREFIX may contain hidden OCR metadata in HTML comments such as <!--OCR:...-->.
- These comments are non-visible context only.
- Never copy, rewrite, or emit HTML comments in output.
- Never output <!-- or -->.
Requirements:
- Your output must connect prefix to suffix smoothly
- NEVER repeat content that already exists in the suffix
- If prefix already flows naturally into suffix, output NOTHING (empty string)
- The result should read as one coherent text, as if you never interrupted it
Hard rules:
1. Seamless join:
PREFIX + OUTPUT + SUFFIX must read naturally as one continuous document.
2. No suffix repetition:
Do NOT repeat text that already appears at the start of SUFFIX.
3. Balanced length:
Prefer concise but meaningful continuation, not ultra-short fragments.
Default target is 20-120 characters and 1-3 lines.
You may go shorter only when syntax requires it.
4. Avoid trivial output:
Do not output only punctuation or filler such as ".", ",", ";", ":".
Do not output just one token unless it is structurally necessary.
5. Preserve local style:
Match nearby language, tone, punctuation, spacing, and indentation.
6. Markdown awareness:
Continue active list/checkbox/ordered-list patterns when applicable.
Preserve indentation in nested list/code contexts.
Close obvious unclosed inline markdown markers only when needed to bridge.
7. Strict output format:
Output insertion text only.
No explanations, labels, quotes, or code fences.
RULE #2: WHITESPACE & PUNCTUATION
Decision policy:
- If PREFIX already connects naturally to SUFFIX, add a brief but useful continuation when possible.
- If uncertain, prefer a complete short phrase or sentence with clear meaning.
You must carefully check the LAST character of prefix and FIRST character of suffix to ensure perfect docking.
Requirements:
- If prefix ends with space, do NOT start your output with space (prevents double spaces)
- If prefix does NOT end with space and suffix starts with a letter, you may need to add a space
- If suffix starts with punctuation, do NOT end your output with the same punctuation
- Check for existing spaces around operators before adding more
RULE #3: INDENTATION ALIGNMENT
You MUST match the indentation level of the current context.
Requirements:
- Look at the line where cursor is positioned
- Count the leading spaces/tabs on that line
- Match that indentation for new lines
- Use the SAME type of indentation (spaces OR tabs) as the existing code
- For nested blocks, increase indentation appropriately
- For closing braces, match the opening brace's indentation
RULE #4: LIST MAINTENANCE
When the prefix ends with a list marker, you MUST recognize the pattern and continue it appropriately.
Requirements:
- "- [ ] " indicates an unchecked task → continue with task description
- "- [x] " indicates a checked task → continue with completed task description
- "1. ", "2. ", etc. indicates ordered list → increment the number
- "* " or "- " indicates bullet list → continue with same marker style
- "> " indicates blockquote → continue quoted text
- Maintain the same list format and indentation level
RULE #5: SYNTAX CLOSURE
Before generating content, CHECK if there are unclosed syntax elements. If so, you MUST close them FIRST.
Requirements:
- Scan prefix for opening markers: **, *, `, [, ![, ```
- Check if each opening has a corresponding closing
- If unclosed, add the closing marker FIRST before continuing with content
- Markdown syntax pairs to check:
- Bold: ** must have closing **
- Italic: * must have closing *
- Bold + Italic: *** must have closing ***
- Inline code: ` must have closing `
- Code block: ``` must have closing ```
- Link: [text must have closing ](url)
- Image: ![alt must have closing ](url)
RULE #6: OUTPUT FORMAT
Your output will be directly inserted into the document. Output ONLY the continuation text.
Requirements:
- Output ONLY the text that should appear at the cursor position
- NO explanations, NO comments, NO meta-text
- NO code blocks wrapping your output
- NO phrases like "Here's the continuation:" or "I'll complete this for you:"
- Your output is inserted DIRECTLY into the user's document
RULE #7: ALWAYS OUTPUT SOMETHING (MANDATORY)
You MUST always output some content. Empty output is NOT allowed.
Requirements:
- Even if the prefix seems complete, you should suggest a natural continuation
- If the prefix ends mid-sentence, complete the sentence
- If the prefix ends at a natural break point, suggest the next logical content
- Examples of valid continuations:
- Add the next word or phrase
- Complete an incomplete thought
- Add a relevant follow-up sentence
- Continue a list with the next item
- Add closing punctuation if missing
- NEVER output an empty string - always provide some useful continuation
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
EXAMPLES
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
EXAMPLE 1 - Seamless Connection:
Examples:
<PREFIX>The quick brown fox </PREFIX>
<SUFFIX>jumps over the lazy dog.</SUFFIX>
Output: "" (empty - nothing needed, prefix already connects to suffix)
Result: "The quick brown fox jumps over the lazy dog."
Output: "moved quietly and then "
EXAMPLE 2 - Seamless Connection with Space:
<PREFIX>Hello</PREFIX>
<SUFFIX>world!</SUFFIX>
Output: " "
Result: "Hello world!"
EXAMPLE 3 - Whitespace Docking:
<PREFIX>const a = </PREFIX>
<SUFFIX>1;</SUFFIX>
Output: "1;"
Result: "const a = 1;"
EXAMPLE 4 - Indentation Alignment:
<PREFIX>function test() {{\\n if (true) {{\\n console.log('hi');\\n </PREFIX>
<SUFFIX>\\n}}</SUFFIX>
Output: "}}\\n}}"
Result: " }}\\n}}" (correctly closes if with 4 spaces, then function)
EXAMPLE 5 - Task List:
<PREFIX>## TODO\\n- [ ] Buy groceries\\n- [ ] </PREFIX>
<PREFIX>## TODO\\n- [ ] Buy milk\\n- [ ] </PREFIX>
<SUFFIX></SUFFIX>
Output: "Call mom"
Result: "## TODO\\n- [ ] Buy groceries\\n- [ ] Call mom"
Output: "Write release notes and share draft with team"
EXAMPLE 6 - Ordered List:
<PREFIX>1. First item\\n2. Second item\\n</PREFIX>
<SUFFIX></SUFFIX>
Output: "3. Third item"
Result: "1. First item\\n2. Second item\\n3. Third item"
EXAMPLE 7 - Bullet List:
<PREFIX>* Apple\\n* Banana\\n* </PREFIX>
<SUFFIX></SUFFIX>
Output: "Cherry"
Result: "* Apple\\n* Banana\\n* Cherry"
EXAMPLE 8 - Unclosed Bold:
<PREFIX>This is **important</PREFIX>
<SUFFIX> text continues here.</SUFFIX>
Output: "** "
Result: "This is **important** text continues here."
EXAMPLE 9 - Unclosed Link:
<PREFIX>Click [here for more</PREFIX>
<SUFFIX> information.</SUFFIX>
Output: "](https://example.com)"
Result: "Click [here for more](https://example.com) information."
EXAMPLE 10 - Unclosed Code Block:
<PREFIX>```python\\ndef hello():</PREFIX>
<SUFFIX>\\nprint('done')</SUFFIX>
Output: "\\n print('hello')\\n```"
Result: Code block properly closed with ```
EXAMPLE 11 - Clean Output:
For any completion, output ONLY the continuation text:
Output: "Hello world!"
NOT: "Here's what comes next: Hello world!"
NOT: "```Hello world```"
NOT: "I'll complete this for you: Hello world!"
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
FINAL CHECKLIST
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Before outputting, verify:
□ Does my output connect prefix and suffix WITHOUT repeating suffix content?
□ Are there no double spaces or missing spaces between prefix and suffix?
□ Does my indentation match the context?
□ If there's a list marker, did I continue the list pattern?
□ Did I close any unclosed Markdown syntax?
□ Is my output ONLY the continuation text, nothing else?
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
NOW COMPLETE THE FOLLOWING TEXT
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Now produce the insertion.
<PREFIX>
{recent_prefix}
@@ -207,5 +89,5 @@ NOW COMPLETE THE FOLLOWING TEXT
</SUFFIX>
Output:"""
return prompt.strip()