ydy0615/llm-in-text
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(editor): add image insertion with OCR support and size limit handling

Browse Source 
Add image button with dropdown menu for uploading local images or inserting from URL.
Integrate VLM-based OCR to extract text context from images and include in AI suggestions.
Implement document size limits to disable AI when exceeding threshold.
Refactor copilot plugin with per-view runtime state and OCR context injection.
Add OCR cache utility for managing image metadata.
Add code splitting configuration for optimized bundle size.
This commit is contained in:
“ydy0615”
2026-02-14 18:28:37 +08:00
parent c64ff7be45
commit 64cfa58376
16 changed files with 1593 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/App.vue
+3 -2
View File
@@ -1,6 +1,7 @@
<script setup>
import MilkdownEditor from './components/MilkdownEditor.vue'
import { ref } from 'vue'
import { defineAsyncComponent, ref } from 'vue'
const MilkdownEditor = defineAsyncComponent(() => import('./components/MilkdownEditor.vue'))
const markdown = ref('')
const emit = defineEmits(['update:markdown'])
src/components/MilkdownEditor.vue
+466 -43
View File
@@ -1,9 +1,15 @@
<template>
<div class="editor-container" ref="containerRef">
<div class="editor-container">
<div ref="root" class="milkdown-editor"></div>
<div class="action-buttons">
<button class="action-btn" @click="triggerUpload">
<button
type="button"
class="action-btn"
aria-label="导入 Markdown 文件"
title="导入 Markdown"
@click="triggerUpload"
>
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/>
<polyline points="17 8 12 3 7 8"/>
@@ -13,7 +19,13 @@
</button>
<input type="file" ref="fileInputRef" @change="handleFileUpload" accept=".md" style="display:none">
<button class="action-btn" @click="exportMarkdown">
<button
type="button"
class="action-btn"
aria-label="导出 Markdown 文件"
title="导出 Markdown"
@click="exportMarkdown"
>
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/>
<polyline points="7 10 12 15 17 10"/>
@@ -22,34 +34,217 @@
<span class="btn-tooltip">导出 Markdown</span>
</button>
<button
class="action-btn ai-toggle"
:class="{ 'ai-disabled': !aiEnabled }"
<div class="image-btn-wrapper">
<button
type="button"
class="action-btn"
aria-label="Insert Image"
title="Insert Image"
@click="toggleImageDropdown"
>
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<rect x="3" y="3" width="18" height="18" rx="2" ry="2"/>
<circle cx="8.5" cy="8.5" r="1.5"/>
<polyline points="21 15 16 10 5 21"/>
</svg>
<span class="btn-tooltip">Insert Image</span>
</button>
<div v-if="showImageDropdown" class="image-dropdown">
<button type="button" @click="triggerImageUpload">Upload Local Image</button>
<button type="button" @click="showUrlDialog = true; showImageDropdown = false">Insert from URL</button>
</div>
</div>
<input type="file" ref="imageInputRef" @change="handleImageUpload" accept="image/*" style="display:none">
<button
type="button"
class="action-btn ai-toggle"
:class="{ 'ai-disabled': !aiEnabled, 'force-disabled': isOverLimit }"
@click="toggleAI"
:disabled="isOverLimit"
:aria-label="aiButtonLabel"
:title="aiButtonLabel"
>
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<path d="M12 2a10 10 0 1 0 10 10A10 10 0 0 0 12 2z"/>
<path d="M12 6v6l4 2"/>
</svg>
<span class="btn-tooltip">{{ aiEnabled ? '禁用 AI' : '启用 AI' }}</span>
<span class="btn-tooltip">{{ aiButtonLabel }}</span>
</button>
<div class="size-indicator" :class="{ 'over-limit': isOverLimit }" aria-live="polite">
{{ sizeInKB }} KB
</div>
</div>
<div v-if="showUrlDialog" class="url-dialog-overlay" @click.self="showUrlDialog = false">
<div class="url-dialog">
<h3>Insert Image from URL</h3>
<input
v-model="imageUrl"
type="url"
placeholder="Enter image URL"
@keyup.enter="insertImageFromUrl"
/>
<div class="url-dialog-buttons">
<button type="button" class="dialog-btn primary" @click="insertImageFromUrl">Insert</button>
<button type="button" class="dialog-btn" @click="showUrlDialog = false; imageUrl = ''">Cancel</button>
</div>
</div>
</div>
</div>
</template>
<script setup>
import { onMounted, onUnmounted, ref } from 'vue'
import { onMounted, onUnmounted, ref, computed } from 'vue'
import { replaceAll } from '@milkdown/kit/utils'
import { Crepe } from '@milkdown/crepe'
import { copilotPlugin, copilotConfigCtx, copilotGhostMark, isCopilotEnabled, setCopilotEnabled, COPILOT_PLUGIN_KEY } from '../plugins/copilotPlugin'
import { editorViewCtx } from '@milkdown/kit/core'
import { copilotPlugin, copilotConfigCtx, copilotGhostMark, setCopilotEnabled, COPILOT_PLUGIN_KEY, SIZE_LIMIT, checkSizeLimit } from '../plugins/copilotPlugin'
import { fetchSuggestion } from '../utils/api.js'
import { DEBUG } from '../utils/config.js'
import { DEBUG, API_URL } from '../utils/config.js'
import { setOcrCache, clearOcrCache, clearAllOcrCache } from '../utils/ocrCache.js'
const emit = defineEmits(['update:markdown'])
const root = ref(null)
const containerRef = ref(null)
const fileInputRef = ref(null)
const imageInputRef = ref(null)
const aiEnabled = ref(true)
const contentSize = ref(0)
const showImageDropdown = ref(false)
const showUrlDialog = ref(false)
const imageUrl = ref('')
const isOverLimit = computed(() => contentSize.value > SIZE_LIMIT)
const sizeInKB = computed(() => Math.floor(contentSize.value / 1024))
const aiButtonLabel = computed(() => {
if (isOverLimit.value) return '文档过大，AI已禁用'
return aiEnabled.value ? '禁用 AI' : '启用 AI'
})
let crepe = null
let markdownSyncTimer = null
const objectUrls = new Set()
const revokeObjectUrl = (url) => {
if (!objectUrls.has(url)) return
URL.revokeObjectURL(url)
objectUrls.delete(url)
clearOcrCache(url)
}
const collectImageObjectUrls = (doc) => {
const activeUrls = new Set()
doc.descendants((node) => {
if (
node.type?.name === 'image' &&
typeof node.attrs?.src === 'string' &&
node.attrs.src.startsWith('blob:')
) {
activeUrls.add(node.attrs.src)
}
})
return activeUrls
}
const syncObjectUrls = (doc) => {
const activeUrls = collectImageObjectUrls(doc)
for (const url of Array.from(objectUrls)) {
if (!activeUrls.has(url)) {
revokeObjectUrl(url)
}
}
}
const refreshSizeAndLimit = (ctx) => {
const view = ctx.get(editorViewCtx)
const { size, overLimit } = checkSizeLimit(view)
contentSize.value = size
if (overLimit && aiEnabled.value) {
aiEnabled.value = false
setCopilotEnabled(view, false)
}
}
const scheduleMarkdownSync = () => {
if (!crepe) return
if (markdownSyncTimer) {
clearTimeout(markdownSyncTimer)
markdownSyncTimer = null
}
markdownSyncTimer = setTimeout(async () => {
markdownSyncTimer = null
if (!crepe) return
try {
let hasGhostSuggestion = false
crepe.editor.action((ctx) => {
const view = ctx.get(editorViewCtx)
const state = COPILOT_PLUGIN_KEY.getState(view.state)
hasGhostSuggestion = Boolean(state?.suggestion && state.from < state.to)
})
// Ghost text is transient UI state and should not leak to emitted markdown.
if (hasGhostSuggestion) return
const markdown = await crepe.getMarkdown()
emit('update:markdown', markdown)
} catch (e) {
if (DEBUG) console.error('[Markdown] Sync failed:', e)
}
}, 120)
}
const clearCurrentSuggestion = (view) => {
const state = COPILOT_PLUGIN_KEY.getState(view.state)
if (state?.suggestion && state.from < state.to) {
const tr = view.state.tr
.delete(state.from, state.to)
.setMeta(COPILOT_PLUGIN_KEY, { from: 0, to: 0, suggestion: '' })
view.dispatch(tr)
}
}
const performOCR = async (file, cacheKey) => {
if (!aiEnabled.value) return
const reader = new FileReader()
reader.onload = async () => {
const dataUrl = typeof reader.result === 'string' ? reader.result : ''
const splitIndex = dataUrl.indexOf(',')
if (splitIndex === -1) return
const base64 = dataUrl.slice(splitIndex + 1)
try {
const ocrUrl = API_URL.replace('/v1/completions', '/v1/ocr')
const res = await fetch(ocrUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
image: base64,
filename: file.name,
language: 'auto'
})
})
const data = await res.json()
if (data.text) {
setOcrCache(cacheKey, data.text)
setOcrCache(file.name, data.text)
if (crepe?.editor) {
crepe.editor.action((ctx) => {
refreshSizeAndLimit(ctx)
})
}
}
} catch (e) {
if (DEBUG) console.error('[OCR] Error:', e)
}
}
reader.readAsDataURL(file)
}
onMounted(async () => {
if (DEBUG) console.log('[Debug] onMounted called')
@@ -61,11 +256,20 @@ onMounted(async () => {
defaultValue: '# Welcome to LLM in text\n\nStart writing your content here...',
features: {
[Crepe.Feature.Latex]: true,
[Crepe.Feature.ImageBlock]: true,
},
featureConfigs: {
[Crepe.Feature.Latex]: {
katexOptions: {},
inlineEditConfirm: 'Escape'
},
[Crepe.Feature.ImageBlock]: {
onUpload: (file) => {
const objectUrl = URL.createObjectURL(file)
objectUrls.add(objectUrl)
performOCR(file, objectUrl)
return objectUrl
}
}
},
config: {
@@ -76,7 +280,7 @@ onMounted(async () => {
crepe.editor.config((ctx) => {
ctx.set(copilotConfigCtx.key, {
fetchSuggestion,
debounceMs: 500
debounceMs: 1000
})
})
@@ -86,24 +290,30 @@ onMounted(async () => {
await crepe.create()
crepe.on((listener) => {
listener.updated((ctx, doc) => {
syncObjectUrls(doc)
refreshSizeAndLimit(ctx)
scheduleMarkdownSync()
})
})
crepe.editor.action((ctx) => {
const view = ctx.get(editorViewCtx)
setCopilotEnabled(view, aiEnabled.value)
refreshSizeAndLimit(ctx)
})
scheduleMarkdownSync()
if (DEBUG) console.log('[Debug] Crepe editor created with copilot plugin')
})
const exportMarkdown = async () => {
if (!crepe) return
const { editorViewCtx } = await import('@milkdown/kit/core')
const { COPILOT_PLUGIN_KEY } = await import('../plugins/copilotPlugin')
crepe.editor.action((ctx) => {
const view = ctx.get(editorViewCtx)
const state = COPILOT_PLUGIN_KEY.getState(view.state)
if (state?.suggestion && state.from < state.to) {
const tr = view.state.tr
.delete(state.from, state.to)
.setMeta(COPILOT_PLUGIN_KEY, { from: 0, to: 0, suggestion: '' })
view.dispatch(tr)
}
clearCurrentSuggestion(view)
})
const markdown = await crepe.getMarkdown()
@@ -112,7 +322,9 @@ const exportMarkdown = async () => {
const a = document.createElement('a')
a.href = url
a.download = `document-${Date.now()}.md`
document.body.appendChild(a)
a.click()
a.remove()
URL.revokeObjectURL(url)
}
@@ -137,28 +349,80 @@ const handleFileUpload = async (event) => {
}
const toggleAI = async () => {
aiEnabled.value = !aiEnabled.value
setCopilotEnabled(aiEnabled.value)
if (isOverLimit.value || !crepe) return
if (!aiEnabled.value && crepe) {
const { editorViewCtx } = await import('@milkdown/kit/core')
aiEnabled.value = !aiEnabled.value
crepe.editor.action((ctx) => {
const view = ctx.get(editorViewCtx)
setCopilotEnabled(view, aiEnabled.value)
if (!aiEnabled.value) {
clearCurrentSuggestion(view)
}
})
}
const toggleImageDropdown = () => {
showImageDropdown.value = !showImageDropdown.value
}
const triggerImageUpload = () => {
showImageDropdown.value = false
imageInputRef.value?.click()
}
const insertImageAtCursor = (src) => {
if (!crepe || !src) return
crepe.editor.action((ctx) => {
const view = ctx.get(editorViewCtx)
const { state } = view
const { schema } = state
crepe.editor.action((ctx) => {
const view = ctx.get(editorViewCtx)
const state = COPILOT_PLUGIN_KEY.getState(view.state)
if (state?.suggestion && state.from < state.to) {
const tr = view.state.tr
.delete(state.from, state.to)
.setMeta(COPILOT_PLUGIN_KEY, { from: 0, to: 0, suggestion: '' })
view.dispatch(tr)
}
})
}
const imageType = schema.nodes.image
if (!imageType) return
const imageNode = imageType.create({ src })
const tr = state.tr.replaceSelectionWith(imageNode)
view.dispatch(tr)
})
}
const handleImageUpload = async (event) => {
const file = event.target.files?.[0]
if (!file) return
const objectUrl = URL.createObjectURL(file)
objectUrls.add(objectUrl)
performOCR(file, objectUrl)
insertImageAtCursor(objectUrl)
event.target.value = ''
}
const insertImageFromUrl = () => {
const url = imageUrl.value.trim()
if (!url) return
insertImageAtCursor(url)
imageUrl.value = ''
showUrlDialog.value = false
}
onUnmounted(() => {
if (markdownSyncTimer) {
clearTimeout(markdownSyncTimer)
markdownSyncTimer = null
}
for (const url of Array.from(objectUrls)) {
revokeObjectUrl(url)
}
clearAllOcrCache()
if (crepe) {
crepe.destroy()
crepe = null
}
})
</script>
@@ -176,6 +440,7 @@ onUnmounted(() => {
bottom: 20px;
right: 20px;
display: flex;
flex-direction: column;
gap: 8px;
z-index: 9999;
}
@@ -193,12 +458,14 @@ onUnmounted(() => {
align-items: center;
justify-content: center;
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
opacity: 0.5;
}
.action-btn:hover {
background-color: #4a90d9;
color: white;
border-color: #4a90d9;
opacity: 1;
}
.action-btn.ai-disabled {
@@ -213,15 +480,42 @@ onUnmounted(() => {
border-color: #4a90d9;
}
.action-btn.force-disabled {
background-color: #ccc;
color: #999;
border-color: #ccc;
cursor: not-allowed;
opacity: 0.6;
}
.action-btn.force-disabled:hover {
background-color: #ccc;
color: #999;
border-color: #ccc;
opacity: 0.6;
}
.size-indicator {
font-size: 10px;
color: #999;
text-align: center;
margin-top: 4px;
}
.size-indicator.over-limit {
color: #e74c3c;
}
.action-btn {
position: relative;
}
.btn-tooltip {
position: absolute;
top: -32px;
left: 50%;
transform: translateX(-50%);
top: 50%;
right: 100%;
transform: translateY(-50%);
margin-right: 8px;
background: #333;
color: #fff;
font-size: 12px;
@@ -237,6 +531,116 @@ onUnmounted(() => {
opacity: 1;
}
.action-btn:focus-visible .btn-tooltip {
opacity: 1;
}
.image-btn-wrapper {
position: relative;
}
.image-dropdown {
position: absolute;
bottom: 100%;
right: 0;
margin-bottom: 8px;
background: #fff;
border: 1px solid #ddd;
border-radius: 8px;
box-shadow: 0 2px 8px rgba(0,0,0,0.15);
overflow: hidden;
z-index: 10000;
min-width: 160px;
}
.image-dropdown button {
display: block;
width: 100%;
padding: 10px 16px;
border: none;
background: none;
text-align: left;
cursor: pointer;
font-size: 14px;
color: #333;
}
.image-dropdown button:hover {
background: #f5f5f5;
}
.url-dialog-overlay {
position: fixed;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: rgba(0,0,0,0.3);
display: flex;
align-items: center;
justify-content: center;
z-index: 10001;
}
.url-dialog {
background: #fff;
padding: 20px;
border-radius: 8px;
box-shadow: 0 4px 16px rgba(0,0,0,0.2);
min-width: 320px;
}
.url-dialog h3 {
margin: 0 0 12px 0;
font-size: 16px;
color: #333;
}
.url-dialog input {
width: 100%;
box-sizing: border-box;
padding: 10px 12px;
border: 1px solid #ddd;
border-radius: 4px;
font-size: 14px;
margin-bottom: 16px;
}
.url-dialog input:focus {
outline: none;
border-color: #4a90d9;
}
.url-dialog-buttons {
display: flex;
justify-content: flex-end;
gap: 8px;
}
.dialog-btn {
padding: 8px 16px;
border: 1px solid #ddd;
border-radius: 4px;
cursor: pointer;
font-size: 14px;
background: #fff;
color: #333;
}
.dialog-btn:hover {
background: #f5f5f5;
}
.dialog-btn.primary {
background: #4a90d9;
color: #fff;
border-color: #4a90d9;
}
.dialog-btn.primary:hover {
background: #3a80c9;
}
.milkdown-editor {
width: 100%;
height: 100%;
@@ -247,7 +651,7 @@ onUnmounted(() => {
.milkdown-editor :deep(.milkdown) {
max-width: none;
margin: 0 !important;
padding: 20px 40px !important;
padding: 0 40px !important;
min-height: 100%;
}
@@ -262,6 +666,25 @@ onUnmounted(() => {
padding: 0 !important;
}
.milkdown-editor :deep(.milkdown > *:first-child) {
margin-top: 0 !important;
padding-top: 0 !important;
}
.milkdown-editor :deep(.ProseMirror) {
margin: 0 !important;
padding: 0 !important;
}
.milkdown-editor :deep(.ProseMirror img) {
max-width: 60%;
height: auto;
}
.milkdown-editor :deep(.ProseMirror > *:first-child) {
margin-top: 0 !important;
}
.milkdown-editor :deep(.milkdown__aside),
.milkdown-editor :deep(.milkdown__aside-wrapper),
.milkdown-editor :deep([class*="aside"]),
@@ -314,7 +737,7 @@ onUnmounted(() => {
.copilot-ghost-text {
color: #999;
opacity: 0.6;
pointer-events: none;
pointer-events: auto;
}
.copilot-ghost-text.copilot-loading {
src/plugins/copilotPlugin.ts
+369 -180
View File
@@ -1,14 +1,14 @@
import { Plugin, PluginKey, Selection } from '@milkdown/prose/state'
import { $prose, $ctx, $markSchema } from '@milkdown/kit/utils'
import { parserCtx } from '@milkdown/kit/core'
import { Node as ProseNode, Fragment, Slice } from '@milkdown/prose/model'
import { Node as ProseNode, Fragment } from '@milkdown/prose/model'
import type { Ctx } from '@milkdown/kit/core'
import type { EditorView } from '@milkdown/prose/view'
import { getOcrCache, checkSizeLimit as checkOcrSizeLimit, OCR_SIZE_LIMIT } from '../utils/ocrCache'
const COPILOT_PLUGIN_KEY = new PluginKey('milkdown-copilot')
const DEBOUNCE_MS = 500
let enabled = true
const DEBOUNCE_MS = 1000
const SIZE_LIMIT = OCR_SIZE_LIMIT
interface CopilotState {
from: number
@@ -21,12 +21,21 @@ interface CopilotConfig {
debounceMs?: number
}
interface CopilotRuntime {
enabled: boolean
debounceTimer: ReturnType<typeof setTimeout> | null
abortController: AbortController | null
ctx: Ctx
}
const initialState: CopilotState = {
from: 0,
to: 0,
suggestion: ''
}
const runtimeByView = new WeakMap<EditorView, CopilotRuntime>()
export const copilotConfigCtx = $ctx<CopilotConfig, 'copilotConfig'>({
fetchSuggestion: async () => '',
debounceMs: DEBOUNCE_MS
@@ -36,21 +45,68 @@ export const copilotGhostMark = $markSchema('copilot_ghost', () => ({
excludes: '_',
inclusive: true,
parseDOM: [{ tag: 'span[data-copilot-ghost]' }],
toDOM: () => ['span', { 'data-copilot-ghost': '', class: 'copilot-ghost-text' }, 0]
toDOM: () => ['span', { 'data-copilot-ghost': '', class: 'copilot-ghost-text' }, 0],
parseMarkdown: {
match: () => false,
runner: () => {}
},
toMarkdown: {
match: (mark) => mark.type.name === 'copilot_ghost',
runner: () => {}
}
}))
let debounceTimer: ReturnType<typeof setTimeout> | null = null
let abortController: AbortController | null = null
let currentCtx: Ctx | null = null
function clearRuntimeRequests(runtime: CopilotRuntime) {
if (runtime.debounceTimer) {
clearTimeout(runtime.debounceTimer)
runtime.debounceTimer = null
}
if (runtime.abortController) {
runtime.abortController.abort()
runtime.abortController = null
}
}
function findGhostRangeByMarks(view: EditorView): { from: number; to: number } | null {
const markType = view.state.schema.marks.copilot_ghost
if (!markType) return null
let from = Number.POSITIVE_INFINITY
let to = -1
view.state.doc.descendants((node, pos) => {
if (node.isText && node.marks.some((m: any) => m.type === markType)) {
from = Math.min(from, pos)
to = Math.max(to, pos + node.nodeSize)
}
return true
})
if (!Number.isFinite(from) || to <= from) return null
return { from, to }
}
function getGhostRange(view: EditorView): { from: number; to: number } | null {
const state = COPILOT_PLUGIN_KEY.getState(view.state)
if (state && state.from < state.to) {
return { from: state.from, to: state.to }
}
return findGhostRangeByMarks(view)
}
function hasGhostText(view: EditorView): boolean {
return getGhostRange(view) !== null
}
function clearGhostText(view: EditorView) {
const state = COPILOT_PLUGIN_KEY.getState(view.state)
if (state && state.suggestion && state.from < state.to) {
const tr = view.state.tr
.delete(state.from, state.to)
.setMeta(COPILOT_PLUGIN_KEY, { ...initialState })
view.dispatch(tr)
}
const range = getGhostRange(view)
if (!range) return
const tr = view.state.tr
.delete(range.from, range.to)
.setMeta(COPILOT_PLUGIN_KEY, { ...initialState })
view.dispatch(tr)
}
function isBlockNode(node: ProseNode): boolean {
@@ -67,39 +123,24 @@ function hasBlockNodes(doc: ProseNode): boolean {
return hasBlock
}
function addGhostMarkToNode(node: ProseNode, ghostMarkType: any): ProseNode {
if (node.isText) {
return node.mark(node.marks.concat(ghostMarkType.create()))
}
if (node.isLeaf) {
return node
}
const newContent: ProseNode[] = []
node.forEach((child) => {
newContent.push(addGhostMarkToNode(child, ghostMarkType))
})
return node.copy(Fragment.from(newContent))
}
function extractInlineContent(doc: ProseNode, ghostMarkType: any, schema: any): Fragment {
function extractInlineContent(doc: ProseNode, schema: any): Fragment {
const nodes: ProseNode[] = []
let isFirstBlock = true
doc.forEach((blockNode) => {
if (!isFirstBlock) {
const hardBreak = schema.nodes.hard_break?.create()
if (hardBreak) {
nodes.push(hardBreak)
} else {
nodes.push(schema.text('\n', [ghostMarkType.create()]))
nodes.push(schema.text('\n'))
}
}
isFirstBlock = false
blockNode.forEach((inlineNode) => {
if (inlineNode.isText) {
const combinedMarks = inlineNode.marks.concat(ghostMarkType.create())
nodes.push(inlineNode.mark(combinedMarks))
nodes.push(inlineNode)
} else if (inlineNode.type.name === 'hard_break') {
nodes.push(inlineNode)
} else if (inlineNode.isLeaf) {
@@ -107,8 +148,7 @@ function extractInlineContent(doc: ProseNode, ghostMarkType: any, schema: any):
} else if (inlineNode.content.size > 0) {
inlineNode.forEach((nestedNode) => {
if (nestedNode.isText) {
const combinedMarks = nestedNode.marks.concat(ghostMarkType.create())
nodes.push(nestedNode.mark(combinedMarks))
nodes.push(nestedNode)
} else if (nestedNode.isLeaf) {
nodes.push(nestedNode)
}
@@ -116,52 +156,83 @@ function extractInlineContent(doc: ProseNode, ghostMarkType: any, schema: any):
}
})
})
return Fragment.from(nodes)
}
async function insertGhostText(view: EditorView, suggestion: string, from: number) {
if (!currentCtx || !suggestion) return
function normalizeSuggestionText(raw: string): string {
if (!raw) return raw
let text = raw.replace(/\r\n?/g, '\n')
const trimmed = text.trim()
// Some models may return a JSON-encoded string literal, decode it if so.
if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
try {
const parsed = JSON.parse(trimmed)
if (typeof parsed === 'string') {
text = parsed.replace(/\r\n?/g, '\n')
}
} catch {
// Keep original text when not valid JSON.
}
}
// If newlines are escaped literally, convert them back.
if (!text.includes('\n') && text.includes('\\n')) {
text = text.replace(/\\n/g, '\n')
}
if (text.includes('\\t')) {
text = text.replace(/\\t/g, '\t')
}
return text
}
async function insertGhostText(view: EditorView, suggestion: string, from: number, ctx: Ctx) {
if (!suggestion) return
const schema = view.state.schema
const markType = schema.marks.copilot_ghost
if (!markType) {
console.error('[Copilot] copilot_ghost mark not found in schema')
return
}
try {
const parser = currentCtx.get(parserCtx)
const parser = ctx.get(parserCtx)
const parsedDoc = await parser(suggestion)
if (!parsedDoc) {
insertPlainText(view, suggestion, from, markType)
return
}
const containsBlocks = hasBlockNodes(parsedDoc)
if (containsBlocks) {
const $from = view.state.doc.resolve(from)
const insertPos = $from.after($from.depth)
const blockNodes: ProseNode[] = []
parsedDoc.forEach((node) => {
blockNodes.push(addGhostMarkToNode(node, markType))
blockNodes.push(node)
})
const fragment = Fragment.from(blockNodes)
const tr = view.state.tr
tr.insert(insertPos, fragment)
const endPos = insertPos + fragment.size
tr.addMark(insertPos, endPos, markType.create())
tr.setMeta(COPILOT_PLUGIN_KEY, { from: insertPos, to: endPos, suggestion })
view.dispatch(tr)
} else {
const inlineFragment = extractInlineContent(parsedDoc, markType, schema)
const inlineFragment = extractInlineContent(parsedDoc, schema)
const tr = view.state.tr
tr.insert(from, inlineFragment)
const endPos = from + inlineFragment.size
tr.addMark(from, endPos, markType.create())
tr.setMeta(COPILOT_PLUGIN_KEY, { from, to: endPos, suggestion })
view.dispatch(tr)
}
@@ -180,66 +251,112 @@ function insertPlainText(view: EditorView, suggestion: string, from: number, mar
view.dispatch(tr)
}
function doFetchSuggestion(view: EditorView, pos: number, prefix: string, suffix: string) {
if (!currentCtx) return
const config = currentCtx.get(copilotConfigCtx.key)
if (abortController) {
abortController.abort()
abortController = null
function extractImageFilenames(doc: ProseNode): string[] {
const filenames: string[] = []
doc.descendants((node: ProseNode) => {
if (node.type.name === 'image' && node.attrs.src) {
filenames.push(node.attrs.src)
}
})
return filenames
}
function buildPrefixWithOCR(prefix: string, doc: ProseNode, cursorPos: number): string {
const ocrEntries: string[] = []
doc.descendants((node: ProseNode, pos) => {
if (pos >= cursorPos) return false
if (node.type.name !== 'image' || !node.attrs.src) return true
const ocrText = getOcrCache(node.attrs.src)
if (!ocrText) return true
const altText = typeof node.attrs.alt === 'string' ? node.attrs.alt : ''
ocrEntries.push(`image(${altText || 'untitled'}): ${ocrText}`)
return true
})
if (!ocrEntries.length) return prefix
return `${prefix}\n\n[OCR Context]\n${ocrEntries.join('\n')}`
}
function doFetchSuggestion(view: EditorView, runtime: CopilotRuntime, pos: number, prefix: string, suffix: string) {
const config = runtime.ctx.get(copilotConfigCtx.key)
if (runtime.abortController) {
runtime.abortController.abort()
runtime.abortController = null
}
abortController = new AbortController()
config.fetchSuggestion(prefix, suffix, abortController.signal)
.then(suggestion => {
if (view.state.selection.from !== pos) return
if (suggestion) {
insertGhostText(view, suggestion, pos)
const controller = new AbortController()
runtime.abortController = controller
config.fetchSuggestion(prefix, suffix, controller.signal)
.then((suggestion) => {
if (!runtime.enabled) return
if (view.state.selection.from !== pos || view.state.selection.to !== pos) return
const normalizedSuggestion = normalizeSuggestionText(suggestion)
if (normalizedSuggestion) {
insertGhostText(view, normalizedSuggestion, pos, runtime.ctx)
}
})
.catch(e => {
if (e.name !== 'AbortError') {
.catch((e: any) => {
if (e?.name !== 'AbortError') {
console.error('[Copilot] Error:', e)
}
})
.finally(() => {
abortController = null
if (runtime.abortController === controller) {
runtime.abortController = null
}
})
}
function scheduleFetch(view: EditorView, pos: number, prefix: string, suffix: string) {
if (!enabled) return
if (debounceTimer) {
clearTimeout(debounceTimer)
debounceTimer = null
function scheduleFetch(view: EditorView, runtime: CopilotRuntime, pos: number, prefix: string, suffix: string) {
if (!runtime.enabled) return
const doc = view.state.doc
const imageFilenames = extractImageFilenames(doc)
const { overLimit } = checkOcrSizeLimit(doc.content.size, imageFilenames)
if (overLimit) {
setCopilotEnabled(view, false)
return
}
debounceTimer = setTimeout(() => {
debounceTimer = null
doFetchSuggestion(view, pos, prefix, suffix)
}, DEBOUNCE_MS)
const prefixWithOCR = buildPrefixWithOCR(prefix, doc, pos)
if (runtime.debounceTimer) {
clearTimeout(runtime.debounceTimer)
runtime.debounceTimer = null
}
const debounceMs = runtime.ctx.get(copilotConfigCtx.key).debounceMs ?? DEBOUNCE_MS
runtime.debounceTimer = setTimeout(() => {
runtime.debounceTimer = null
doFetchSuggestion(view, runtime, pos, prefixWithOCR, suffix)
}, debounceMs)
}
function acceptSuggestion(view: EditorView) {
const state = COPILOT_PLUGIN_KEY.getState(view.state)
if (!state?.suggestion || state.from >= state.to) return false
const range = getGhostRange(view)
if (!range) return false
const tr = view.state.tr
const doc = tr.doc
const from = state.from
const to = state.to
const from = range.from
const to = range.to
const markType = view.state.schema.marks.copilot_ghost
if (!markType) return false
doc.nodesBetween(from, to, (node, pos) => {
if (node.marks.some((m: any) => m.type.name === 'copilot_ghost')) {
tr.removeMark(pos, pos + node.nodeSize, view.state.schema.marks.copilot_ghost)
if (node.marks.some((m: any) => m.type === markType)) {
tr.removeMark(pos, pos + node.nodeSize, markType)
}
})
const endPos = Math.min(state.to, tr.doc.content.size)
const endPos = Math.min(to, tr.doc.content.size)
tr.setSelection(Selection.near(tr.doc.resolve(endPos)))
tr.setMeta(COPILOT_PLUGIN_KEY, { ...initialState })
view.dispatch(tr)
@@ -247,108 +364,180 @@ function acceptSuggestion(view: EditorView) {
}
function rejectSuggestion(view: EditorView) {
const state = COPILOT_PLUGIN_KEY.getState(view.state)
if (!state?.suggestion) return false
if (!hasGhostText(view)) return false
clearGhostText(view)
return true
}
export const copilotPlugin = $prose((ctx) => {
currentCtx = ctx
return new Plugin<CopilotState>({
key: COPILOT_PLUGIN_KEY,
state: {
init: () => ({ ...initialState }),
apply: (tr, value) => {
const meta = tr.getMeta(COPILOT_PLUGIN_KEY)
if (meta !== undefined) {
return meta
}
if (tr.docChanged && value.suggestion) {
return { ...initialState }
}
return value
export const copilotPlugin = $prose((ctx) => new Plugin<CopilotState>({
key: COPILOT_PLUGIN_KEY,
state: {
init: () => ({ ...initialState }),
apply: (tr, value) => {
const meta = tr.getMeta(COPILOT_PLUGIN_KEY)
if (meta !== undefined) {
return meta
}
},
props: {
handleKeyDown: (view, event) => {
const state = COPILOT_PLUGIN_KEY.getState(view.state)
if (event.key === 'Tab' && state?.suggestion) {
event.preventDefault()
return acceptSuggestion(view)
}
if (event.key === 'Escape' && state?.suggestion) {
event.preventDefault()
return rejectSuggestion(view)
}
if (state?.suggestion && event.key !== 'Shift' && event.key !== 'Control' && event.key !== 'Alt' && event.key !== 'Meta') {
clearGhostText(view)
}
return false
},
handleClick: (view, pos) => {
const state = COPILOT_PLUGIN_KEY.getState(view.state)
if (!state?.suggestion) return false
if (pos >= state.from && pos < state.to) {
return acceptSuggestion(view)
}
if (tr.docChanged && value.suggestion) {
return { ...initialState }
}
return value
}
},
props: {
handleKeyDown: (view, event) => {
const hasGhost = hasGhostText(view)
if (event.key === 'Tab' && hasGhost) {
event.preventDefault()
return acceptSuggestion(view)
}
if (event.key === 'Escape' && hasGhost) {
event.preventDefault()
return rejectSuggestion(view)
}
if (hasGhost && event.key !== 'Shift' && event.key !== 'Control' && event.key !== 'Alt' && event.key !== 'Meta') {
clearGhostText(view)
return false
}
return false
},
view: () => ({
update: (view, prevState) => {
if (view.state.doc.eq(prevState.doc) && view.state.selection.eq(prevState.selection)) {
return
}
const state = COPILOT_PLUGIN_KEY.getState(view.state)
if (state?.suggestion) {
return
}
if (!view.state.doc.eq(prevState.doc)) {
const { from, to } = view.state.selection
if (from !== to) return
const doc = view.state.doc
const prefix = doc.textBetween(0, from)
const suffix = doc.textBetween(to, doc.content.size)
scheduleFetch(view, from, prefix, suffix)
}
handleClick: (view, pos) => {
const range = getGhostRange(view)
if (!range) return false
if (pos >= range.from && pos <= range.to) {
return acceptSuggestion(view)
}
})
})
})
clearGhostText(view)
return false
}
},
view: (view) => {
let activeView = view
let activeDom = view.dom
const runtime: CopilotRuntime = {
enabled: true,
debounceTimer: null,
abortController: null,
ctx
}
runtimeByView.set(view, runtime)
const onKeydownCapture = (event: KeyboardEvent) => {
if (!hasGhostText(activeView)) return
if (event.key === 'Tab') {
event.preventDefault()
event.stopPropagation()
event.stopImmediatePropagation?.()
acceptSuggestion(activeView)
return
}
if (event.key === 'Escape') {
event.preventDefault()
event.stopPropagation()
event.stopImmediatePropagation?.()
rejectSuggestion(activeView)
}
}
const onPointerDownCapture = (event: MouseEvent) => {
if (!hasGhostText(activeView)) return
const targetNode = event.target instanceof Node ? event.target : null
const target = targetNode instanceof Element ? targetNode : targetNode?.parentElement
if (!target) return
// Accept suggestion when user clicks any rendered ghost-text fragment.
if (target.closest('[data-copilot-ghost]')) {
event.preventDefault()
event.stopPropagation()
event.stopImmediatePropagation?.()
acceptSuggestion(activeView)
}
}
const bindDomListeners = (dom: HTMLElement) => {
dom.addEventListener('keydown', onKeydownCapture, true)
dom.addEventListener('mousedown', onPointerDownCapture, true)
}
const unbindDomListeners = (dom: HTMLElement) => {
dom.removeEventListener('keydown', onKeydownCapture, true)
dom.removeEventListener('mousedown', onPointerDownCapture, true)
}
bindDomListeners(activeDom)
return {
update: (nextView, prevState) => {
if (nextView.dom !== activeDom) {
unbindDomListeners(activeDom)
activeDom = nextView.dom
bindDomListeners(activeDom)
}
activeView = nextView
const docChanged = !nextView.state.doc.eq(prevState.doc)
const selectionChanged = !nextView.state.selection.eq(prevState.selection)
if (!docChanged && !selectionChanged) {
return
}
if (hasGhostText(nextView)) {
return
}
const { from, to } = nextView.state.selection
if (from !== to) {
clearRuntimeRequests(runtime)
return
}
const doc = nextView.state.doc
const prefix = doc.textBetween(0, from)
const suffix = doc.textBetween(to, doc.content.size)
scheduleFetch(nextView, runtime, from, prefix, suffix)
},
destroy: () => {
unbindDomListeners(activeDom)
clearRuntimeRequests(runtime)
runtimeByView.delete(view)
}
}
}
}))
export { COPILOT_PLUGIN_KEY }
export function isCopilotEnabled(): boolean {
return enabled
export function isCopilotEnabled(view: EditorView): boolean {
return runtimeByView.get(view)?.enabled ?? true
}
export function setCopilotEnabled(value: boolean): void {
enabled = value
export function setCopilotEnabled(view: EditorView, value: boolean): void {
const runtime = runtimeByView.get(view)
if (!runtime) return
runtime.enabled = value
if (!value) {
if (debounceTimer) {
clearTimeout(debounceTimer)
debounceTimer = null
}
if (abortController) {
abortController.abort()
abortController = null
}
clearRuntimeRequests(runtime)
}
}
export function checkSizeLimit(view: EditorView): { size: number; overLimit: boolean } {
const doc = view.state.doc
const imageFilenames = extractImageFilenames(doc)
const result = checkOcrSizeLimit(doc.content.size, imageFilenames)
return { size: result.size, overLimit: result.overLimit }
}
export { SIZE_LIMIT }
src/style.css
+2 -1
View File
@@ -72,5 +72,6 @@ body {
padding: 0;
width: 100%;
height: 100%;
overflow: hidden;
overflow-x: hidden;
overflow-y: auto;
}
src/utils/ocrCache.js
+45
View File
@@ -0,0 +1,45 @@
const SIZE_LIMIT = 64 * 1024
const ocrCache = new Map()
export function setOcrCache(filename, text) {
ocrCache.set(filename, text)
}
export function getOcrCache(filename) {
return ocrCache.get(filename) || ''
}
export function clearOcrCache(filename) {
ocrCache.delete(filename)
}
export function hasOcrCache(filename) {
return ocrCache.has(filename)
}
export function clearAllOcrCache() {
ocrCache.clear()
}
export function calculateOcrSize(imageFilenames) {
let total = 0
for (const name of imageFilenames) {
const text = ocrCache.get(name)
if (text) total += new Blob([text]).size
}
return total
}
export function checkSizeLimit(docTextSize, imageFilenames) {
const ocrSize = calculateOcrSize(imageFilenames)
const total = docTextSize + ocrSize
return {
size: total,
docSize: docTextSize,
ocrSize: ocrSize,
overLimit: total > SIZE_LIMIT
}
}
export const OCR_SIZE_LIMIT = SIZE_LIMIT