export const DOC_BLOCK_NODE_TYPE = 'doc_block' export const DOC_BLOCK_FENCE_LANG = 'llm-file' export const DOC_CONTEXT_LIMIT = 32 * 1024 const IMAGE_MD_RE = /!\[[^\]]*]\([^)]+\)/g const IMAGE_HTML_RE = /]*>/gi const HEADER_SEPARATOR = '\n---\n' const FENCED_DOC_BLOCK_RE = /(^|\n)(`{3,})llm-file[^\n]*\n([\s\S]*?)\n\2(?=\n|$)/g const LEGACY_DOC_BLOCK_RE = /[\s\S]*?<\/doc_end>/g function normalizeMarkdownText(markdown = '') { return String(markdown || '').replace(/\r\n?/g, '\n') } function clipDocContext(content = '', limit = 0) { if (!limit || content.length <= limit) return content return `${content.slice(0, limit)}...` } const AUDIO_EXT_RE = /\.(wav|mp3|m4a|ogg|flac)$/i const VIDEO_EXT_RE = /\.(mp4|webm|mov|avi|mkv|m4v|ogv)$/i export function normalizeDocType(value = '') { const lower = String(value || '').trim().toLowerCase() if (lower === 'txt' || lower === 'text' || lower === 'plain') return 'txt' if (lower === 'json') return 'json' if (lower === 'toml') return 'toml' if (lower === 'yaml' || lower === 'yml') return 'yaml' if (lower === 'doc' || lower === 'docx' || lower === 'word') return 'docx' if (lower === 'ppt' || lower === 'pptx' || lower === 'powerpoint') return 'pptx' if (lower === 'pdf') return 'pdf' // Audio types - map to their extensions for doc block display if (lower === 'wav' || lower === 'wave') return 'wav' if (lower === 'mp3' || lower === 'mpeg') return 'mp3' if (lower === 'm4a' || lower === 'aac') return 'm4a' if (lower === 'ogg' || lower === 'opus') return 'ogg' if (lower === 'flac') return 'flac' return 'txt' } export function getDocTypeFromFilename(name = '') { const lower = String(name || '').toLowerCase() if (lower.endsWith('.docx')) return 'docx' if (lower.endsWith('.pptx')) return 'pptx' if (lower.endsWith('.pdf')) return 'pdf' if (lower.endsWith('.json')) return 'json' if (lower.endsWith('.toml')) return 'toml' if (lower.endsWith('.yaml') || lower.endsWith('.yml')) return 'yaml' // Audio types - preserve the actual extension for display if (lower.endsWith('.wav')) return 'wav' if (lower.endsWith('.mp3') || lower.endsWith('.mpeg')) return 'mp3' if (lower.endsWith('.m4a') || lower.endsWith('.aac')) return 'm4a' if (lower.endsWith('.ogg') || lower.endsWith('.opus')) return 'ogg' if (lower.endsWith('.flac')) return 'flac' return 'txt' } export function isAudioFile(file) { if (!file) return false const name = String(file.name || '').toLowerCase() const type = String(file.type || '').toLowerCase() return AUDIO_EXT_RE.test(name) || type.startsWith('audio/') } export function isVideoFile(file) { if (!file) return false const name = String(file.name || '').toLowerCase() const type = String(file.type || '').toLowerCase() return VIDEO_EXT_RE.test(name) || type.startsWith('video/') } export function isSupportedDocFile(file) { if (!file) return false const name = String(file.name || '').toLowerCase() const type = String(file.type || '').toLowerCase() return ( name.endsWith('.txt') || name.endsWith('.json') || name.endsWith('.toml') || name.endsWith('.yaml') || name.endsWith('.yml') || name.endsWith('.docx') || name.endsWith('.pptx') || name.endsWith('.pdf') || AUDIO_EXT_RE.test(name) || VIDEO_EXT_RE.test(name) || type === 'text/plain' || type === 'application/json' || type === 'text/yaml' || type === 'text/x-yaml' || type === 'application/x-yaml' || type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' || type === 'application/vnd.openxmlformats-officedocument.presentationml.presentation' || type === 'application/pdf' || type.startsWith('audio/') || type.startsWith('video/') ) } export function sanitizeDocContent(markdown = '') { return normalizeMarkdownText(markdown) .replace(IMAGE_MD_RE, '') .replace(IMAGE_HTML_RE, '') } function quoteMeta(value = '') { return JSON.stringify(String(value ?? '')) } function parseMetaLine(line = '') { const idx = line.indexOf(':') if (idx < 0) return null const key = line.slice(0, idx).trim() const rawValue = line.slice(idx + 1).trim() if (!key) return null try { return [key, JSON.parse(rawValue)] } catch { return [key, rawValue] } } function pickFence(content = '') { const matches = String(content || '').match(/`{3,}/g) || [] const maxLen = matches.reduce((max, item) => Math.max(max, item.length), 2) return '`'.repeat(maxLen + 1) } export function buildDocBlockValue(attrs = {}) { const docType = normalizeDocType(attrs.docType) const docName = String(attrs.docName || `document.${docType}`) const uploadTime = String(attrs.uploadTime || new Date().toISOString()) const collapsed = Boolean(attrs.collapsed) const content = sanitizeDocContent(attrs.content || '') return [ `type: ${quoteMeta(docType)}`, `name: ${quoteMeta(docName)}`, `uploadTime: ${quoteMeta(uploadTime)}`, `collapsed: ${collapsed ? 'true' : 'false'}`, '---', content, ].join('\n') } export function parseDocBlockValue(raw = '') { const normalized = normalizeMarkdownText(raw) const separatorIndex = normalized.indexOf(HEADER_SEPARATOR) const headerText = separatorIndex >= 0 ? normalized.slice(0, separatorIndex) : '' const bodyText = separatorIndex >= 0 ? normalized.slice(separatorIndex + HEADER_SEPARATOR.length) : normalized const attrs = { docType: 'txt', docName: 'document.txt', uploadTime: '', collapsed: false, content: sanitizeDocContent(bodyText), } for (const line of headerText.split('\n')) { const parsed = parseMetaLine(line) if (!parsed) continue const [key, value] = parsed if (key === 'type') attrs.docType = normalizeDocType(value) if (key === 'name' && value) attrs.docName = String(value) if (key === 'uploadTime' && value) attrs.uploadTime = String(value) if (key === 'collapsed') attrs.collapsed = value === true || value === 'true' } if (!attrs.docName) attrs.docName = `document.${attrs.docType}` return attrs } export function buildDocBlockMarkdown(attrs = {}) { const value = buildDocBlockValue(attrs) const fence = pickFence(value) return `${fence}${DOC_BLOCK_FENCE_LANG}\n${value}\n${fence}` } export function buildDocContextFence(attrs = {}) { const docType = normalizeDocType(attrs.docType) const content = sanitizeDocContent(attrs.content || '') const fence = pickFence(content) return `${fence}${docType}\n${content}\n${fence}` } export function buildLegacyDocBlock(attrs = {}) { const docType = normalizeDocType(attrs.docType) const docName = String(attrs.docName || `document.${docType}`) const uploadTime = String(attrs.uploadTime || new Date().toISOString()) const content = sanitizeDocContent(attrs.content || '') return `\n${content}\n` } export function parseLegacyDocBlock(raw = '') { const normalized = normalizeMarkdownText(raw) const match = normalized.match(/^\n?([\s\S]*?)\n?<\/doc_end>$/) if (!match) return null return { docType: normalizeDocType(match[1]), docName: match[2] || 'document.txt', uploadTime: match[3] || '', collapsed: match[4] === 'true', content: sanitizeDocContent(match[5] || ''), } } export function extractDocBlockContextFromMarkdown(markdown = '', contentLimit = 0) { const normalized = normalizeMarkdownText(markdown) const contexts = [] const appendContext = (attrs) => { if (!attrs) return const rawContent = String(attrs.content || '') if (!rawContent.trim()) return contexts.push(buildDocContextFence({ docType: attrs.docType, content: clipDocContext(rawContent, contentLimit), })) } normalized.replace(FENCED_DOC_BLOCK_RE, (_full, _prefix, _fence, value) => { appendContext(parseDocBlockValue(value)) return _full }) normalized.replace(LEGACY_DOC_BLOCK_RE, (full) => { appendContext(parseLegacyDocBlock(full)) return full }) return contexts.join('\n\n') } export function transformDocBlockMarkdownForClipboard(markdown = '') { const normalized = normalizeMarkdownText(markdown) const replacedFence = normalized.replace(FENCED_DOC_BLOCK_RE, (full, prefix, _fence, value) => { const attrs = parseDocBlockValue(value) return `${prefix}${buildDocContextFence(attrs)}` }) return replacedFence.replace(LEGACY_DOC_BLOCK_RE, (full) => { const attrs = parseLegacyDocBlock(full) return attrs ? buildDocContextFence(attrs) : full }) } export function stripDocBlockMarkdown(markdown = '') { return normalizeMarkdownText(markdown).replace(FENCED_DOC_BLOCK_RE, '$1') } export function transformLegacyDocBlocksForExport(markdown = '') { return normalizeMarkdownText(markdown).replace(LEGACY_DOC_BLOCK_RE, (full) => { const attrs = parseLegacyDocBlock(full) return attrs ? buildDocBlockMarkdown(attrs) : full }) } export function transformSpecialDocBlocksToLegacy(markdown = '') { return normalizeMarkdownText(markdown).replace(FENCED_DOC_BLOCK_RE, (full, prefix, _fence, value) => { const attrs = parseDocBlockValue(value) return `${prefix}${buildLegacyDocBlock(attrs)}` }) }