export const DOC_BLOCK_NODE_TYPE = 'doc_block'
export const DOC_BLOCK_FENCE_LANG = 'llm-file'
export const DOC_CONTEXT_LIMIT = 32 * 1024
const IMAGE_MD_RE = /!\[[^\]]*]\([^)]+\)/g
const IMAGE_HTML_RE = /
]*>/gi
const HEADER_SEPARATOR = '\n---\n'
const FENCED_DOC_BLOCK_RE = /(^|\n)(`{3,})llm-file[^\n]*\n([\s\S]*?)\n\2(?=\n|$)/g
const LEGACY_DOC_BLOCK_RE = /[\s\S]*?<\/doc_end>/g
function normalizeMarkdownText(markdown = '') {
return String(markdown || '').replace(/\r\n?/g, '\n')
}
function clipDocContext(content = '', limit = 0) {
if (!limit || content.length <= limit) return content
return `${content.slice(0, limit)}...`
}
const AUDIO_EXT_RE = /\.(wav|mp3|m4a|ogg|flac)$/i
const VIDEO_EXT_RE = /\.(mp4|webm|mov|avi|mkv|m4v|ogv)$/i
export function normalizeDocType(value = '') {
const lower = String(value || '').trim().toLowerCase()
if (lower === 'txt' || lower === 'text' || lower === 'plain') return 'txt'
if (lower === 'json') return 'json'
if (lower === 'toml') return 'toml'
if (lower === 'yaml' || lower === 'yml') return 'yaml'
if (lower === 'doc' || lower === 'docx' || lower === 'word') return 'docx'
if (lower === 'ppt' || lower === 'pptx' || lower === 'powerpoint') return 'pptx'
if (lower === 'pdf') return 'pdf'
// Audio types - map to their extensions for doc block display
if (lower === 'wav' || lower === 'wave') return 'wav'
if (lower === 'mp3' || lower === 'mpeg') return 'mp3'
if (lower === 'm4a' || lower === 'aac') return 'm4a'
if (lower === 'ogg' || lower === 'opus') return 'ogg'
if (lower === 'flac') return 'flac'
return 'txt'
}
export function getDocTypeFromFilename(name = '') {
const lower = String(name || '').toLowerCase()
if (lower.endsWith('.docx')) return 'docx'
if (lower.endsWith('.pptx')) return 'pptx'
if (lower.endsWith('.pdf')) return 'pdf'
if (lower.endsWith('.json')) return 'json'
if (lower.endsWith('.toml')) return 'toml'
if (lower.endsWith('.yaml') || lower.endsWith('.yml')) return 'yaml'
// Audio types - preserve the actual extension for display
if (lower.endsWith('.wav')) return 'wav'
if (lower.endsWith('.mp3') || lower.endsWith('.mpeg')) return 'mp3'
if (lower.endsWith('.m4a') || lower.endsWith('.aac')) return 'm4a'
if (lower.endsWith('.ogg') || lower.endsWith('.opus')) return 'ogg'
if (lower.endsWith('.flac')) return 'flac'
return 'txt'
}
export function isAudioFile(file) {
if (!file) return false
const name = String(file.name || '').toLowerCase()
const type = String(file.type || '').toLowerCase()
return AUDIO_EXT_RE.test(name) || type.startsWith('audio/')
}
export function isVideoFile(file) {
if (!file) return false
const name = String(file.name || '').toLowerCase()
const type = String(file.type || '').toLowerCase()
return VIDEO_EXT_RE.test(name) || type.startsWith('video/')
}
export function isSupportedDocFile(file) {
if (!file) return false
const name = String(file.name || '').toLowerCase()
const type = String(file.type || '').toLowerCase()
return (
name.endsWith('.txt') ||
name.endsWith('.json') ||
name.endsWith('.toml') ||
name.endsWith('.yaml') ||
name.endsWith('.yml') ||
name.endsWith('.docx') ||
name.endsWith('.pptx') ||
name.endsWith('.pdf') ||
AUDIO_EXT_RE.test(name) ||
VIDEO_EXT_RE.test(name) ||
type === 'text/plain' ||
type === 'application/json' ||
type === 'text/yaml' ||
type === 'text/x-yaml' ||
type === 'application/x-yaml' ||
type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' ||
type === 'application/vnd.openxmlformats-officedocument.presentationml.presentation' ||
type === 'application/pdf' ||
type.startsWith('audio/') ||
type.startsWith('video/')
)
}
export function sanitizeDocContent(markdown = '') {
return normalizeMarkdownText(markdown)
.replace(IMAGE_MD_RE, '')
.replace(IMAGE_HTML_RE, '')
}
function quoteMeta(value = '') {
return JSON.stringify(String(value ?? ''))
}
function parseMetaLine(line = '') {
const idx = line.indexOf(':')
if (idx < 0) return null
const key = line.slice(0, idx).trim()
const rawValue = line.slice(idx + 1).trim()
if (!key) return null
try {
return [key, JSON.parse(rawValue)]
} catch {
return [key, rawValue]
}
}
function pickFence(content = '') {
const matches = String(content || '').match(/`{3,}/g) || []
const maxLen = matches.reduce((max, item) => Math.max(max, item.length), 2)
return '`'.repeat(maxLen + 1)
}
export function buildDocBlockValue(attrs = {}) {
const docType = normalizeDocType(attrs.docType)
const docName = String(attrs.docName || `document.${docType}`)
const uploadTime = String(attrs.uploadTime || new Date().toISOString())
const collapsed = Boolean(attrs.collapsed)
const content = sanitizeDocContent(attrs.content || '')
return [
`type: ${quoteMeta(docType)}`,
`name: ${quoteMeta(docName)}`,
`uploadTime: ${quoteMeta(uploadTime)}`,
`collapsed: ${collapsed ? 'true' : 'false'}`,
'---',
content,
].join('\n')
}
export function parseDocBlockValue(raw = '') {
const normalized = normalizeMarkdownText(raw)
const separatorIndex = normalized.indexOf(HEADER_SEPARATOR)
const headerText = separatorIndex >= 0 ? normalized.slice(0, separatorIndex) : ''
const bodyText = separatorIndex >= 0 ? normalized.slice(separatorIndex + HEADER_SEPARATOR.length) : normalized
const attrs = {
docType: 'txt',
docName: 'document.txt',
uploadTime: '',
collapsed: false,
content: sanitizeDocContent(bodyText),
}
for (const line of headerText.split('\n')) {
const parsed = parseMetaLine(line)
if (!parsed) continue
const [key, value] = parsed
if (key === 'type') attrs.docType = normalizeDocType(value)
if (key === 'name' && value) attrs.docName = String(value)
if (key === 'uploadTime' && value) attrs.uploadTime = String(value)
if (key === 'collapsed') attrs.collapsed = value === true || value === 'true'
}
if (!attrs.docName) attrs.docName = `document.${attrs.docType}`
return attrs
}
export function buildDocBlockMarkdown(attrs = {}) {
const value = buildDocBlockValue(attrs)
const fence = pickFence(value)
return `${fence}${DOC_BLOCK_FENCE_LANG}\n${value}\n${fence}`
}
export function buildDocContextFence(attrs = {}) {
const docType = normalizeDocType(attrs.docType)
const content = sanitizeDocContent(attrs.content || '')
const fence = pickFence(content)
return `${fence}${docType}\n${content}\n${fence}`
}
export function buildLegacyDocBlock(attrs = {}) {
const docType = normalizeDocType(attrs.docType)
const docName = String(attrs.docName || `document.${docType}`)
const uploadTime = String(attrs.uploadTime || new Date().toISOString())
const content = sanitizeDocContent(attrs.content || '')
return `\n${content}\n`
}
export function parseLegacyDocBlock(raw = '') {
const normalized = normalizeMarkdownText(raw)
const match = normalized.match(/^\n?([\s\S]*?)\n?<\/doc_end>$/)
if (!match) return null
return {
docType: normalizeDocType(match[1]),
docName: match[2] || 'document.txt',
uploadTime: match[3] || '',
collapsed: match[4] === 'true',
content: sanitizeDocContent(match[5] || ''),
}
}
export function extractDocBlockContextFromMarkdown(markdown = '', contentLimit = 0) {
const normalized = normalizeMarkdownText(markdown)
const contexts = []
const appendContext = (attrs) => {
if (!attrs) return
const rawContent = String(attrs.content || '')
if (!rawContent.trim()) return
contexts.push(buildDocContextFence({
docType: attrs.docType,
content: clipDocContext(rawContent, contentLimit),
}))
}
normalized.replace(FENCED_DOC_BLOCK_RE, (_full, _prefix, _fence, value) => {
appendContext(parseDocBlockValue(value))
return _full
})
normalized.replace(LEGACY_DOC_BLOCK_RE, (full) => {
appendContext(parseLegacyDocBlock(full))
return full
})
return contexts.join('\n\n')
}
export function transformDocBlockMarkdownForClipboard(markdown = '') {
const normalized = normalizeMarkdownText(markdown)
const replacedFence = normalized.replace(FENCED_DOC_BLOCK_RE, (full, prefix, _fence, value) => {
const attrs = parseDocBlockValue(value)
return `${prefix}${buildDocContextFence(attrs)}`
})
return replacedFence.replace(LEGACY_DOC_BLOCK_RE, (full) => {
const attrs = parseLegacyDocBlock(full)
return attrs ? buildDocContextFence(attrs) : full
})
}
export function stripDocBlockMarkdown(markdown = '') {
return normalizeMarkdownText(markdown).replace(FENCED_DOC_BLOCK_RE, '$1')
}
export function transformLegacyDocBlocksForExport(markdown = '') {
return normalizeMarkdownText(markdown).replace(LEGACY_DOC_BLOCK_RE, (full) => {
const attrs = parseLegacyDocBlock(full)
return attrs ? buildDocBlockMarkdown(attrs) : full
})
}
export function transformSpecialDocBlocksToLegacy(markdown = '') {
return normalizeMarkdownText(markdown).replace(FENCED_DOC_BLOCK_RE, (full, prefix, _fence, value) => {
const attrs = parseDocBlockValue(value)
return `${prefix}${buildLegacyDocBlock(attrs)}`
})
}