feat(copilot): enhance OCR handling with inline tags and document serializer
- Replace HTML comment OCR metadata with inline `<OCR:...>` tags - Implement serializer-based markdown conversion for prefix/suffix content - Add extractTextFromOCR utility function for text extraction - Enable Table, Diagram, and ListCheck features in MilkdownEditor - Add periodic debug logging for document state analysis
This commit is contained in:
+105
-19
@@ -1,14 +1,15 @@
|
||||
import { Plugin, PluginKey, Selection } from '@milkdown/prose/state'
|
||||
import { $prose, $ctx, $markSchema } from '@milkdown/kit/utils'
|
||||
import { parserCtx } from '@milkdown/kit/core'
|
||||
import { parserCtx, serializerCtx } from '@milkdown/kit/core'
|
||||
import { Node as ProseNode, Fragment } from '@milkdown/prose/model'
|
||||
import type { Ctx } from '@milkdown/kit/core'
|
||||
import type { EditorView } from '@milkdown/prose/view'
|
||||
import { getOcrCache, checkSizeLimit as checkOcrSizeLimit, OCR_SIZE_LIMIT } from '../utils/ocrCache'
|
||||
import { getOcrCache, checkSizeLimit as checkOcrSizeLimit, OCR_SIZE_LIMIT, extractTextFromOCR } from '../utils/ocrCache'
|
||||
|
||||
const COPILOT_PLUGIN_KEY = new PluginKey('milkdown-copilot')
|
||||
const DEBOUNCE_MS = 1000
|
||||
const SIZE_LIMIT = OCR_SIZE_LIMIT
|
||||
const DEBUG = true
|
||||
const IMAGE_NODE_TYPES = new Set(['image', 'image-block', 'imageBlock'])
|
||||
|
||||
interface CopilotState {
|
||||
@@ -279,27 +280,57 @@ function extractImageFilenames(doc: ProseNode): string[] {
|
||||
return filenames
|
||||
}
|
||||
|
||||
function buildPrefixWithOCR(prefix: string, doc: ProseNode, cursorPos: number): string {
|
||||
const ocrEntries: string[] = []
|
||||
function buildPrefixWithOCRFromMarkdown(
|
||||
doc: ProseNode,
|
||||
cursorPos: number,
|
||||
prefixMarkdown: string,
|
||||
serializer: any,
|
||||
schema: any
|
||||
): string {
|
||||
const imageNodes: Array<{pos: number, src: string, label: string}> = []
|
||||
|
||||
doc.descendants((node: ProseNode, pos) => {
|
||||
if (pos >= cursorPos) return false
|
||||
if (!isImageNodeWithSrc(node)) return true
|
||||
|
||||
if (!isImageNodeWithSrc(node)) return pos < cursorPos
|
||||
const src = getImageSrc(node)
|
||||
const ocrText = getOcrCache(src)
|
||||
if (!ocrText) return true
|
||||
|
||||
const label = getImageLabel(node)
|
||||
const safeOcrText = ocrText.replace(/<!--|-->/g, '').trim()
|
||||
if (!safeOcrText) return true
|
||||
|
||||
ocrEntries.push(`image(${label}): ${safeOcrText}`)
|
||||
return true
|
||||
imageNodes.push({ pos, src, label })
|
||||
return pos < cursorPos
|
||||
})
|
||||
|
||||
if (!ocrEntries.length) return prefix
|
||||
return `${prefix}\n\n<!--OCR:\n${ocrEntries.join('\n')}\n-->`
|
||||
if (imageNodes.length === 0) {
|
||||
return prefixMarkdown
|
||||
}
|
||||
|
||||
imageNodes.sort((a, b) => a.pos - b.pos)
|
||||
|
||||
const parts: string[] = []
|
||||
let lastPos = 0
|
||||
|
||||
for (const img of imageNodes) {
|
||||
if (img.pos > lastPos) {
|
||||
const slice = doc.slice(lastPos, img.pos)
|
||||
const sliceDoc = schema.topNodeType.createAndFill(undefined, slice.content)
|
||||
parts.push(sliceDoc ? serializer(sliceDoc) : doc.textBetween(lastPos, img.pos))
|
||||
}
|
||||
const imageSyntax = ``
|
||||
parts.push(imageSyntax)
|
||||
const ocrText = getOcrCache(img.src)
|
||||
if (ocrText) {
|
||||
const textOnly = extractTextFromOCR(ocrText, 100)
|
||||
if (textOnly) {
|
||||
parts.push(` <OCR:${textOnly}>`)
|
||||
}
|
||||
}
|
||||
lastPos = img.pos + 1
|
||||
}
|
||||
|
||||
if (lastPos < cursorPos) {
|
||||
const slice = doc.slice(lastPos, cursorPos)
|
||||
const sliceDoc = schema.topNodeType.createAndFill(undefined, slice.content)
|
||||
parts.push(sliceDoc ? serializer(sliceDoc) : doc.textBetween(lastPos, cursorPos))
|
||||
}
|
||||
|
||||
return parts.join('')
|
||||
}
|
||||
|
||||
function doFetchSuggestion(view: EditorView, runtime: CopilotRuntime, pos: number, prefix: string, suffix: string) {
|
||||
@@ -339,6 +370,7 @@ function scheduleFetch(view: EditorView, runtime: CopilotRuntime, pos: number, p
|
||||
if (!runtime.enabled) return
|
||||
|
||||
const doc = view.state.doc
|
||||
const schema = view.state.schema
|
||||
const imageFilenames = extractImageFilenames(doc)
|
||||
const { overLimit } = checkOcrSizeLimit(doc.content.size, imageFilenames)
|
||||
|
||||
@@ -347,7 +379,61 @@ function scheduleFetch(view: EditorView, runtime: CopilotRuntime, pos: number, p
|
||||
return
|
||||
}
|
||||
|
||||
const prefixWithOCR = buildPrefixWithOCR(prefix, doc, pos)
|
||||
const serializer = runtime.ctx.get(serializerCtx)
|
||||
|
||||
// 尝试使用 serializer 将文档切片转换为 Markdown
|
||||
let prefixMarkdown = ''
|
||||
let suffixMarkdown = ''
|
||||
|
||||
try {
|
||||
// 方法1: 使用 slice 创建文档节点
|
||||
const prefixSlice = doc.slice(0, pos)
|
||||
if (prefixSlice.content.size > 0) {
|
||||
const prefixDoc = schema.topNodeType.createAndFill(undefined, prefixSlice.content)
|
||||
if (prefixDoc) {
|
||||
prefixMarkdown = serializer(prefixDoc)
|
||||
}
|
||||
}
|
||||
if (!prefixMarkdown) {
|
||||
// 方法2: 直接序列化整个文档然后截取
|
||||
const fullMarkdown = serializer(doc)
|
||||
const fullDoc = view.state.doc
|
||||
const totalLen = fullDoc.content.size
|
||||
if (totalLen > 0 && pos < totalLen) {
|
||||
// 简单估算位置
|
||||
prefixMarkdown = fullMarkdown.substring(0, Math.floor(fullMarkdown.length * pos / totalLen))
|
||||
}
|
||||
}
|
||||
if (!prefixMarkdown) {
|
||||
// 回退到 textBetween 但添加换行符
|
||||
prefixMarkdown = doc.textBetween(0, pos, '\n', '\n')
|
||||
}
|
||||
|
||||
// Suffix
|
||||
const suffixSlice = doc.slice(pos)
|
||||
if (suffixSlice.content.size > 0) {
|
||||
const suffixDoc = schema.topNodeType.createAndFill(undefined, suffixSlice.content)
|
||||
if (suffixDoc) {
|
||||
suffixMarkdown = serializer(suffixDoc)
|
||||
}
|
||||
}
|
||||
if (!suffixMarkdown) {
|
||||
suffixMarkdown = doc.textBetween(pos, doc.content.size, '\n', '\n')
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[Copilot] Serializer error:', e)
|
||||
prefixMarkdown = doc.textBetween(0, pos, '\n', '\n')
|
||||
suffixMarkdown = doc.textBetween(pos, doc.content.size, '\n', '\n')
|
||||
}
|
||||
|
||||
const prefixWithOCR = buildPrefixWithOCRFromMarkdown(doc, pos, prefixMarkdown, serializer, schema)
|
||||
|
||||
if (DEBUG) {
|
||||
console.log('[Copilot] ===== LLM Request =====')
|
||||
console.log('[Copilot] PREFIX:', prefixWithOCR)
|
||||
console.log('[Copilot] SUFFIX:', suffixMarkdown)
|
||||
console.log('[Copilot] ======================')
|
||||
}
|
||||
|
||||
if (runtime.debounceTimer) {
|
||||
clearTimeout(runtime.debounceTimer)
|
||||
@@ -357,7 +443,7 @@ function scheduleFetch(view: EditorView, runtime: CopilotRuntime, pos: number, p
|
||||
const debounceMs = runtime.ctx.get(copilotConfigCtx.key).debounceMs ?? DEBOUNCE_MS
|
||||
runtime.debounceTimer = setTimeout(() => {
|
||||
runtime.debounceTimer = null
|
||||
doFetchSuggestion(view, runtime, pos, prefixWithOCR, suffix)
|
||||
doFetchSuggestion(view, runtime, pos, prefixWithOCR, suffixMarkdown)
|
||||
}, debounceMs)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user