feat(copilot): enhance OCR handling with inline tags and document serializer

- Replace HTML comment OCR metadata with inline `<OCR:...>` tags
- Implement serializer-based markdown conversion for prefix/suffix content
- Add extractTextFromOCR utility function for text extraction
- Enable Table, Diagram, and ListCheck features in MilkdownEditor
- Add periodic debug logging for document state analysis
This commit is contained in:
2026-02-14 23:53:26 +08:00
parent 794fbf8493
commit 03bb21d5c6
5 changed files with 3496 additions and 225 deletions
+105 -19
View File
@@ -1,14 +1,15 @@
import { Plugin, PluginKey, Selection } from '@milkdown/prose/state'
import { $prose, $ctx, $markSchema } from '@milkdown/kit/utils'
import { parserCtx } from '@milkdown/kit/core'
import { parserCtx, serializerCtx } from '@milkdown/kit/core'
import { Node as ProseNode, Fragment } from '@milkdown/prose/model'
import type { Ctx } from '@milkdown/kit/core'
import type { EditorView } from '@milkdown/prose/view'
import { getOcrCache, checkSizeLimit as checkOcrSizeLimit, OCR_SIZE_LIMIT } from '../utils/ocrCache'
import { getOcrCache, checkSizeLimit as checkOcrSizeLimit, OCR_SIZE_LIMIT, extractTextFromOCR } from '../utils/ocrCache'
const COPILOT_PLUGIN_KEY = new PluginKey('milkdown-copilot')
const DEBOUNCE_MS = 1000
const SIZE_LIMIT = OCR_SIZE_LIMIT
const DEBUG = true
const IMAGE_NODE_TYPES = new Set(['image', 'image-block', 'imageBlock'])
interface CopilotState {
@@ -279,27 +280,57 @@ function extractImageFilenames(doc: ProseNode): string[] {
return filenames
}
function buildPrefixWithOCR(prefix: string, doc: ProseNode, cursorPos: number): string {
const ocrEntries: string[] = []
function buildPrefixWithOCRFromMarkdown(
doc: ProseNode,
cursorPos: number,
prefixMarkdown: string,
serializer: any,
schema: any
): string {
const imageNodes: Array<{pos: number, src: string, label: string}> = []
doc.descendants((node: ProseNode, pos) => {
if (pos >= cursorPos) return false
if (!isImageNodeWithSrc(node)) return true
if (!isImageNodeWithSrc(node)) return pos < cursorPos
const src = getImageSrc(node)
const ocrText = getOcrCache(src)
if (!ocrText) return true
const label = getImageLabel(node)
const safeOcrText = ocrText.replace(/<!--|-->/g, '').trim()
if (!safeOcrText) return true
ocrEntries.push(`image(${label}): ${safeOcrText}`)
return true
imageNodes.push({ pos, src, label })
return pos < cursorPos
})
if (!ocrEntries.length) return prefix
return `${prefix}\n\n<!--OCR:\n${ocrEntries.join('\n')}\n-->`
if (imageNodes.length === 0) {
return prefixMarkdown
}
imageNodes.sort((a, b) => a.pos - b.pos)
const parts: string[] = []
let lastPos = 0
for (const img of imageNodes) {
if (img.pos > lastPos) {
const slice = doc.slice(lastPos, img.pos)
const sliceDoc = schema.topNodeType.createAndFill(undefined, slice.content)
parts.push(sliceDoc ? serializer(sliceDoc) : doc.textBetween(lastPos, img.pos))
}
const imageSyntax = `![${img.label}](${img.src})`
parts.push(imageSyntax)
const ocrText = getOcrCache(img.src)
if (ocrText) {
const textOnly = extractTextFromOCR(ocrText, 100)
if (textOnly) {
parts.push(` <OCR:${textOnly}>`)
}
}
lastPos = img.pos + 1
}
if (lastPos < cursorPos) {
const slice = doc.slice(lastPos, cursorPos)
const sliceDoc = schema.topNodeType.createAndFill(undefined, slice.content)
parts.push(sliceDoc ? serializer(sliceDoc) : doc.textBetween(lastPos, cursorPos))
}
return parts.join('')
}
function doFetchSuggestion(view: EditorView, runtime: CopilotRuntime, pos: number, prefix: string, suffix: string) {
@@ -339,6 +370,7 @@ function scheduleFetch(view: EditorView, runtime: CopilotRuntime, pos: number, p
if (!runtime.enabled) return
const doc = view.state.doc
const schema = view.state.schema
const imageFilenames = extractImageFilenames(doc)
const { overLimit } = checkOcrSizeLimit(doc.content.size, imageFilenames)
@@ -347,7 +379,61 @@ function scheduleFetch(view: EditorView, runtime: CopilotRuntime, pos: number, p
return
}
const prefixWithOCR = buildPrefixWithOCR(prefix, doc, pos)
const serializer = runtime.ctx.get(serializerCtx)
// 尝试使用 serializer 将文档切片转换为 Markdown
let prefixMarkdown = ''
let suffixMarkdown = ''
try {
// 方法1: 使用 slice 创建文档节点
const prefixSlice = doc.slice(0, pos)
if (prefixSlice.content.size > 0) {
const prefixDoc = schema.topNodeType.createAndFill(undefined, prefixSlice.content)
if (prefixDoc) {
prefixMarkdown = serializer(prefixDoc)
}
}
if (!prefixMarkdown) {
// 方法2: 直接序列化整个文档然后截取
const fullMarkdown = serializer(doc)
const fullDoc = view.state.doc
const totalLen = fullDoc.content.size
if (totalLen > 0 && pos < totalLen) {
// 简单估算位置
prefixMarkdown = fullMarkdown.substring(0, Math.floor(fullMarkdown.length * pos / totalLen))
}
}
if (!prefixMarkdown) {
// 回退到 textBetween 但添加换行符
prefixMarkdown = doc.textBetween(0, pos, '\n', '\n')
}
// Suffix
const suffixSlice = doc.slice(pos)
if (suffixSlice.content.size > 0) {
const suffixDoc = schema.topNodeType.createAndFill(undefined, suffixSlice.content)
if (suffixDoc) {
suffixMarkdown = serializer(suffixDoc)
}
}
if (!suffixMarkdown) {
suffixMarkdown = doc.textBetween(pos, doc.content.size, '\n', '\n')
}
} catch (e) {
console.error('[Copilot] Serializer error:', e)
prefixMarkdown = doc.textBetween(0, pos, '\n', '\n')
suffixMarkdown = doc.textBetween(pos, doc.content.size, '\n', '\n')
}
const prefixWithOCR = buildPrefixWithOCRFromMarkdown(doc, pos, prefixMarkdown, serializer, schema)
if (DEBUG) {
console.log('[Copilot] ===== LLM Request =====')
console.log('[Copilot] PREFIX:', prefixWithOCR)
console.log('[Copilot] SUFFIX:', suffixMarkdown)
console.log('[Copilot] ======================')
}
if (runtime.debounceTimer) {
clearTimeout(runtime.debounceTimer)
@@ -357,7 +443,7 @@ function scheduleFetch(view: EditorView, runtime: CopilotRuntime, pos: number, p
const debounceMs = runtime.ctx.get(copilotConfigCtx.key).debounceMs ?? DEBOUNCE_MS
runtime.debounceTimer = setTimeout(() => {
runtime.debounceTimer = null
doFetchSuggestion(view, runtime, pos, prefixWithOCR, suffix)
doFetchSuggestion(view, runtime, pos, prefixWithOCR, suffixMarkdown)
}, debounceMs)
}