From b55af1eff0cd0000c1d49ed77e5ed974c0246632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cydy0615=E2=80=9D?= <“allenyuan410@gmail.com”> Date: Sat, 6 Jun 2026 17:18:15 +0800 Subject: [PATCH] feat: add Docker support and update backend dependencies - Introduced `requirements.docker.txt` for Docker-specific dependencies. - Updated `requirements.txt` to include `psycopg[binary]` and `python-multipart`. - Enhanced test suite in `test_main_endpoints.py` to cover document CRUD operations. - Modified `docker-compose.yml` to include PostgreSQL and frontend services. - Added Nginx configuration for reverse proxying API requests. - Refactored file handling in Vue components to support new document storage backend. - Created new utility functions in `docsApi.js` for document management. - Updated configuration to support new API endpoints for document operations. - Adjusted Vite configuration to proxy API requests to the local backend. --- .dockerignore | 8 + .env.example | 6 + AGENTS.md | 18 +- Dockerfile.frontend | 20 + README.md | 32 ++ backend/.env.example | 19 +- backend/Dockerfile | 16 + backend/docs_store.py | 668 +++++++++++++++++++++++++++ backend/main.py | 148 +++++- backend/requirements.docker.txt | 10 + backend/requirements.txt | 2 + backend/tests/test_main_endpoints.py | 69 +++ docker-compose.yml | 67 ++- docker/nginx.conf | 21 + src/components/FileContent.vue | 36 +- src/composables/useFileSystem.js | 525 +++++++-------------- src/utils/config.js | 11 +- src/utils/docsApi.js | 120 +++++ src/views/DocsView.vue | 22 +- vite.config.js | 2 +- 20 files changed, 1413 insertions(+), 407 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile.frontend create mode 100644 backend/Dockerfile create mode 100644 backend/docs_store.py create mode 100644 backend/requirements.docker.txt create mode 100644 docker/nginx.conf create mode 100644 src/utils/docsApi.js diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..14aae47 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +node_modules +dist +htmlcov +.pytest_cache +.git +docker-data +backend/__pycache__ +backend/tests/__pycache__ diff --git a/.env.example b/.env.example index f8f22e0..3ce0d8d 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,12 @@ VITE_TTS_STATUS_URL= VITE_TTS_CONFIG_URL= VITE_ASR_URL= VITE_JOB_LOAD_URL= +VITE_DOCS_NODES_URL= +VITE_DOCS_FOLDERS_URL= +VITE_DOCS_TEXT_FILES_URL= +VITE_DOCS_UPLOAD_URL= +VITE_DOCS_BLOB_BASE_URL= +VITE_DOCS_NODES_BASE_URL= VITE_PRO_FRONTEND_TIMEOUT_MS=3660000 # Document block compression context limit (characters) diff --git a/AGENTS.md b/AGENTS.md index 7f54013..84fc56c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ - 这是一个智能 Markdown 编辑器，前端负责编辑器 UI、上传导出、补全交互和设置状态，后端负责 LLM、OCR、文件转换和 TTS 接口。 - 前端技术栈：Vue 3 + Vite + Milkdown/Crepe + Pinia + Vue Router。 -- 后端技术栈：FastAPI + Python + Ollama。 +- 后端技术栈：FastAPI + Python + OpenAI-compatible LLM endpoint。 ## 功能块系统（核心概念） @@ -73,6 +73,21 @@ - pytest backend/tests/test_prompt.py -v - pytest backend/tests/test_llm.py -v +## Docker 部署约定 + +- 本机部署目录固定在 `/Volumes/New Volume/lit/` 下，不在仓库外再散落数据库或 Docker 持久化目录。 +- 当前推荐的部署工作目录是 `/Volumes/New Volume/lit/llm-in-text/`；把仓库同步到该目录后，从该目录执行 `docker compose up -d --build`。 +- Docker 持久化数据统一落在部署目录内的 `docker-data/`，包括 PostgreSQL、Redis 和任务共享临时目录。 +- 容器内访问宿主机模型服务时，不要继续使用 `localhost`；应改成 `host.docker.internal` 之类的容器可达地址。 +- 当前 Docker 部署默认使用轻量后端依赖集（`backend/requirements.docker.txt`），覆盖补全、OCR、转换、文档空间和队列，不默认包含本地 `torch` / TTS / ASR 模型栈。 +- 修改 Docker 相关文件时，除了代码本身，还要同步检查： + - `docker-compose.yml` + - `backend/Dockerfile` + - `backend/requirements.docker.txt` + - `Dockerfile.frontend` + - `docker/nginx.conf` + - `backend/.env.example` 与实际部署用 `backend/.env` + ## 代码约定 - 不要把整个仓库当成“全小写+短横线命名”项目。当前实际情况是： @@ -120,4 +135,3 @@ - README.md 对产品功能有参考价值，但其中补全、TTS/ASR 和部分接口说明已经比代码旧。 - backend/TTS_ASR_MACOS_FIX.md 和 backend/tests/TESTING_GUIDE.md 更适合作为历史背景，不应在与代码冲突时被当成事实来源。 - 修改行为时，优先参考实现代码和对应测试，再决定是否同步普通文档。 - diff --git a/Dockerfile.frontend b/Dockerfile.frontend new file mode 100644 index 0000000..284bd42 --- /dev/null +++ b/Dockerfile.frontend @@ -0,0 +1,20 @@ +ARG DOCKER_REGISTRY_PREFIX= +FROM ${DOCKER_REGISTRY_PREFIX}node:22-alpine AS build + +WORKDIR /app + +COPY package.json package-lock.json ./ +RUN npm ci + +COPY index.html vite.config.js ./ +COPY public ./public +COPY src ./src + +RUN npm run build + +FROM ${DOCKER_REGISTRY_PREFIX}nginx:1.27-alpine + +COPY docker/nginx.conf /etc/nginx/conf.d/default.conf +COPY --from=build /app/dist /usr/share/nginx/html + +EXPOSE 80 diff --git a/README.md b/README.md index 567213a..9543bb1 100644 --- a/README.md +++ b/README.md @@ -68,12 +68,44 @@ - 后端: python backend/main.py (端口8001) - 前端: npm run dev (端口5173) +## Docker 部署 + +将整个项目目录放进 New Volume 的 `lit` 文件夹后，在项目根目录执行： + +```bash +cp backend/.env.example backend/.env +docker compose up -d --build +``` + +默认对外端口： +- 前端: `http://localhost:8080` +- 后端: `http://localhost:8001` + +持久化目录全部位于当前项目下的 `docker-data/`： +- PostgreSQL: `docker-data/postgres` +- Redis: `docker-data/redis` +- 任务共享临时目录: `docker-data/jobs` + +部署前至少需要修改这些环境变量： +- `backend/.env` 中的 `LLM_BASE_URL` +- `backend/.env` 中的 `LLM_API_KEY` +- `backend/.env` 或 shell 环境中的 `DATABASE_URL` +- `backend/.env` 中的 `API_KEY` + ## API接口 - POST /v1/completions 流式补全建议 - POST /v1/ocr 图片文字识别 - POST /v1/convert 文档转换 - POST /v1/completions/cancel 取消请求 +- GET /v1/docs/nodes 文档空间节点列表 +- POST /v1/docs/folders 创建文件夹 +- POST /v1/docs/files/text 创建文本文件 +- POST /v1/docs/files/upload 上传文件到文档空间 +- PATCH /v1/docs/nodes/{id} 更新节点 +- PUT /v1/docs/files/{id}/blob 替换文件二进制内容 +- DELETE /v1/docs/nodes/{id} 删除节点 +- GET /v1/docs/files/{id}/blob 下载或预览原文件 - GET /v1/tts-asr/status TTS/ASR模型状态 - GET /v1/tts-asr/config TTS/ASR配置信息 - POST /v1/tts-asr/warmup 模型预热 diff --git a/backend/.env.example b/backend/.env.example index 878ff2d..d6065a4 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,16 +1,15 @@ -# LLM provider (OpenAI-compatible endpoint) -LLM_BASE_URL=http://localhost:11434/v1/ -# For Ollama, API key is not required but a placeholder is needed. -LLM_API_KEY=ollama +# OpenAI-compatible endpoint +LLM_BASE_URL=https://api.openai.com/v1/ +LLM_API_KEY=sk-your-key -# Default model for inline completions (e.g., gpt-oss:20b, qwen3:8b) -LLM_MODEL=gpt-oss:20b +# Default model for inline completions +LLM_MODEL=gpt-4.1-mini # Pro-tier model (defaults to LLM_MODEL if unset) -PRO_LLM_MODEL=gpt-oss:20b +PRO_LLM_MODEL=gpt-4.1 -# Vision model for OCR (e.g., qwen3-vl:30b, llava) -VLM_MODEL=qwen3-vl:30b +# Vision model for OCR +VLM_MODEL=gpt-4.1-mini # API key for the FastAPI app (change in production) API_KEY=your-secret-key-here @@ -18,6 +17,8 @@ API_KEY=your-secret-key-here # Job backend JOB_BACKEND=redis REDIS_URL=redis://localhost:6379/0 +DATABASE_URL=postgresql://llm_in_text:llm_in_text_change_me@localhost:5432/llm_in_text +DOCS_BACKEND=postgres JOB_REDIS_PREFIX=llmtext:jobs JOB_CONSUMER_NAME= JOB_SHARED_TEMP_DIR=/tmp/llm-in-text-jobs diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..7be90a7 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,16 @@ +ARG DOCKER_REGISTRY_PREFIX= +FROM ${DOCKER_REGISTRY_PREFIX}python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +WORKDIR /app/backend + +COPY backend/requirements.docker.txt /tmp/requirements.docker.txt +RUN pip install --no-cache-dir -r /tmp/requirements.docker.txt + +COPY backend /app/backend + +EXPOSE 8001 + +CMD ["python", "main.py"] diff --git a/backend/docs_store.py b/backend/docs_store.py new file mode 100644 index 0000000..95a9736 --- /dev/null +++ b/backend/docs_store.py @@ -0,0 +1,668 @@ +import mimetypes +import os +import threading +import uuid +from dataclasses import dataclass +from typing import Any + +try: # pragma: no cover - optional in some test paths + import psycopg + from psycopg.rows import dict_row +except Exception: # pragma: no cover + psycopg = None + dict_row = None + + +MAX_TEXT_SIZE = 8 * 1024 * 1024 +PREVIEW_TEXT_SIZE = 2 * 1024 * 1024 + +TEXT_EXTENSIONS = { + "md", "markdown", "txt", "json", "js", "jsx", "ts", "tsx", + "css", "scss", "less", "html", "htm", "py", "vue", "xml", + "yaml", "yml", "csv", "log", "sql", "toml", "ini", "cfg", + "conf", "sh", "bat", "ps1", "java", "c", "cpp", "h", "hpp", + "go", "rs", "swift", "kt", "rb", "php", "pl", "r", "scala", + "gradle", "properties", "env", "gitignore", "dockerfile", +} + +BINARY_EXTENSIONS = { + "exe", "dll", "so", "dylib", "bin", "dat", "obj", "o", "a", + "doc", "docx", "xls", "xlsx", "ppt", "pptx", "odt", "ods", "odp", + "pdf", "zip", "rar", "7z", "tar", "gz", "bz2", "xz", + "png", "jpg", "jpeg", "gif", "bmp", "ico", "webp", "svg", + "mp3", "mp4", "wav", "avi", "mov", "mkv", "flv", "wmv", + "ttf", "otf", "woff", "woff2", "eot", + "class", "pyc", "pyo", "jar", "war", "ear", + "db", "sqlite", "mdb", "accdb", + "pem", "key", "crt", "cer", "p12", "pfx", "jks", + "msg", "eml", "pst", "ost", + "dwg", "dxf", "step", "stl", "fbx", "3ds", "blend", +} + +DEFAULT_MIME_TYPES = { + "avi": "video/x-msvideo", + "md": "text/markdown", + "markdown": "text/markdown", + "mkv": "video/x-matroska", + "mov": "video/quicktime", + "mp4": "video/mp4", + "txt": "text/plain", + "json": "application/json", + "js": "text/javascript", + "jsx": "text/javascript", + "ts": "text/typescript", + "tsx": "text/typescript", + "css": "text/css", + "html": "text/html", + "htm": "text/html", + "py": "text/x-python", + "vue": "text/plain", + "xml": "application/xml", + "yaml": "text/yaml", + "yml": "text/yaml", + "csv": "text/csv", + "log": "text/plain", + "sql": "text/plain", + "toml": "text/plain", + "ini": "text/plain", + "cfg": "text/plain", + "conf": "text/plain", + "sh": "text/plain", + "bat": "text/plain", + "ps1": "text/plain", + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "flv": "video/x-flv", + "m4v": "video/x-m4v", + "png": "image/png", + "gif": "image/gif", + "webp": "image/webp", + "svg": "image/svg+xml", + "pdf": "application/pdf", + "ogg": "video/ogg", + "ogv": "video/ogg", + "webm": "video/webm", + "wmv": "video/x-ms-wmv", +} + + +def _now_sql() -> str: + return "CURRENT_TIMESTAMP" + + +def get_extension(name: str = "") -> str: + value = str(name or "") + if "." not in value: + return value.lower() if value.lower() == "dockerfile" else "" + return value.rsplit(".", 1)[-1].lower() + + +def infer_mime_type(name: str, fallback: str = "") -> str: + if fallback: + return fallback + ext = get_extension(name) + guessed = DEFAULT_MIME_TYPES.get(ext) + if guessed: + return guessed + guessed, _ = mimetypes.guess_type(name) + return guessed or "application/octet-stream" + + +def is_text_file(name: str, mime_type: str = "") -> bool: + ext = get_extension(name) + if ext in BINARY_EXTENSIONS: + return False + mime_value = str(mime_type or "").lower() + return ext in TEXT_EXTENSIONS or mime_value.startswith("text/") or "json" in mime_value or "xml" in mime_value + + +def _text_payload(text: str) -> dict[str, Any]: + encoded = text.encode("utf-8") + preview = encoded[:PREVIEW_TEXT_SIZE].decode("utf-8", errors="ignore") + return { + "storage_kind": "text", + "size": len(encoded), + "content_text": text, + "preview_text": preview if len(encoded) > PREVIEW_TEXT_SIZE else text, + "is_truncated_preview": len(encoded) > PREVIEW_TEXT_SIZE, + "blob_data": None, + } + + +def prepare_file_payload(name: str, raw_bytes: bytes, mime_type: str = "") -> dict[str, Any]: + resolved_mime = infer_mime_type(name, mime_type) + if is_text_file(name, resolved_mime): + text = raw_bytes.decode("utf-8", errors="ignore") + return { + "mime_type": resolved_mime, + **_text_payload(text), + } + return { + "mime_type": resolved_mime, + "storage_kind": "blob", + "size": len(raw_bytes), + "content_text": "", + "preview_text": "", + "is_truncated_preview": False, + "blob_data": raw_bytes, + } + + +@dataclass +class BlobPayload: + content: bytes + mime_type: str + filename: str + + +class BaseDocumentStore: + _UNSET = object() + + def list_nodes(self) -> list[dict[str, Any]]: + raise NotImplementedError + + def create_folder(self, name: str, parent_id: str | None) -> dict[str, Any]: + raise NotImplementedError + + def create_text_file(self, name: str, parent_id: str | None, content: str = "") -> dict[str, Any]: + raise NotImplementedError + + def upload_file(self, name: str, parent_id: str | None, raw_bytes: bytes, mime_type: str = "") -> dict[str, Any]: + raise NotImplementedError + + def update_node(self, node_id: str, *, name: str | None | object = _UNSET, parent_id: str | None | object = _UNSET, content: str | None | object = _UNSET) -> dict[str, Any]: + raise NotImplementedError + + def replace_blob(self, node_id: str, filename: str, raw_bytes: bytes, mime_type: str = "") -> dict[str, Any]: + raise NotImplementedError + + def delete_node(self, node_id: str) -> None: + raise NotImplementedError + + def get_blob(self, node_id: str) -> BlobPayload: + raise NotImplementedError + + +class InMemoryDocumentStore(BaseDocumentStore): + def __init__(self) -> None: + self.nodes: dict[str, dict[str, Any]] = {} + + def _serialize(self, node: dict[str, Any]) -> dict[str, Any]: + return { + "id": node["id"], + "parentId": node["parent_id"], + "type": node["type"], + "name": node["name"], + "mimeType": node.get("mime_type") or "", + "storageKind": node.get("storage_kind") or "text", + "size": int(node.get("size") or 0), + "content": node.get("content_text") or "", + "previewText": node.get("preview_text") or "", + "isTruncatedPreview": bool(node.get("is_truncated_preview")), + "createdAt": int(node["created_at"]), + "updatedAt": int(node["updated_at"]), + } + + def _timestamp(self) -> int: + import time + return int(time.time() * 1000) + + def _get(self, node_id: str) -> dict[str, Any]: + node = self.nodes.get(node_id) + if node is None: + raise KeyError(node_id) + return node + + def list_nodes(self) -> list[dict[str, Any]]: + ordered = sorted( + self.nodes.values(), + key=lambda item: (item["type"] != "folder", item["name"].lower(), item["created_at"]), + ) + return [self._serialize(node) for node in ordered] + + def create_folder(self, name: str, parent_id: str | None) -> dict[str, Any]: + now = self._timestamp() + node = { + "id": str(uuid.uuid4()), + "parent_id": parent_id, + "type": "folder", + "name": name, + "mime_type": "", + "storage_kind": "text", + "size": 0, + "content_text": "", + "preview_text": "", + "is_truncated_preview": False, + "blob_data": None, + "created_at": now, + "updated_at": now, + } + self.nodes[node["id"]] = node + return self._serialize(node) + + def create_text_file(self, name: str, parent_id: str | None, content: str = "") -> dict[str, Any]: + now = self._timestamp() + payload = _text_payload(content) + node = { + "id": str(uuid.uuid4()), + "parent_id": parent_id, + "type": "file", + "name": name, + "mime_type": infer_mime_type(name), + **payload, + "created_at": now, + "updated_at": now, + } + self.nodes[node["id"]] = node + return self._serialize(node) + + def upload_file(self, name: str, parent_id: str | None, raw_bytes: bytes, mime_type: str = "") -> dict[str, Any]: + now = self._timestamp() + payload = prepare_file_payload(name, raw_bytes, mime_type) + node = { + "id": str(uuid.uuid4()), + "parent_id": parent_id, + "type": "file", + "name": name, + **payload, + "created_at": now, + "updated_at": now, + } + self.nodes[node["id"]] = node + return self._serialize(node) + + def update_node(self, node_id: str, *, name: str | None | object = BaseDocumentStore._UNSET, parent_id: str | None | object = BaseDocumentStore._UNSET, content: str | None | object = BaseDocumentStore._UNSET) -> dict[str, Any]: + node = self._get(node_id) + if name is not BaseDocumentStore._UNSET: + node["name"] = name + if node["type"] == "file": + node["mime_type"] = infer_mime_type(name, node.get("mime_type") or "") + if parent_id is not BaseDocumentStore._UNSET: + node["parent_id"] = parent_id + if content is not BaseDocumentStore._UNSET: + payload = _text_payload(content) + node.update(payload) + node["updated_at"] = self._timestamp() + return self._serialize(node) + + def replace_blob(self, node_id: str, filename: str, raw_bytes: bytes, mime_type: str = "") -> dict[str, Any]: + node = self._get(node_id) + payload = prepare_file_payload(filename, raw_bytes, mime_type) + node.update(payload) + node["name"] = filename + node["mime_type"] = payload["mime_type"] + node["updated_at"] = self._timestamp() + return self._serialize(node) + + def delete_node(self, node_id: str) -> None: + descendants = {node_id} + changed = True + while changed: + changed = False + for node in list(self.nodes.values()): + if node.get("parent_id") in descendants and node["id"] not in descendants: + descendants.add(node["id"]) + changed = True + for current_id in descendants: + self.nodes.pop(current_id, None) + + def get_blob(self, node_id: str) -> BlobPayload: + node = self._get(node_id) + if node["type"] != "file": + raise FileNotFoundError(node_id) + if node.get("blob_data") is not None: + content = node["blob_data"] + else: + content = (node.get("content_text") or "").encode("utf-8") + return BlobPayload( + content=content, + mime_type=node.get("mime_type") or infer_mime_type(node.get("name") or ""), + filename=node["name"], + ) + + +class PostgresDocumentStore(BaseDocumentStore): + def __init__(self, database_url: str) -> None: + if psycopg is None or dict_row is None: + raise RuntimeError("psycopg 未安装，无法使用 PostgreSQL 文档存储") + self.database_url = database_url + self._init_lock = threading.Lock() + self._initialized = False + + def _connect(self): + return psycopg.connect(self.database_url, autocommit=True, row_factory=dict_row) + + def _ensure_initialized(self) -> None: + if self._initialized: + return + with self._init_lock: + if self._initialized: + return + with self._connect() as conn: + with conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE IF NOT EXISTS document_nodes ( + id TEXT PRIMARY KEY, + parent_id TEXT REFERENCES document_nodes(id) ON DELETE CASCADE, + type TEXT NOT NULL CHECK (type IN ('folder', 'file')), + name TEXT NOT NULL, + mime_type TEXT NOT NULL DEFAULT '', + storage_kind TEXT NOT NULL DEFAULT 'text', + size BIGINT NOT NULL DEFAULT 0, + content_text TEXT NOT NULL DEFAULT '', + preview_text TEXT NOT NULL DEFAULT '', + is_truncated_preview BOOLEAN NOT NULL DEFAULT FALSE, + blob_data BYTEA, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS document_nodes_parent_idx ON document_nodes(parent_id)" + ) + self._initialized = True + + def _serialize(self, row: dict[str, Any]) -> dict[str, Any]: + created = row["created_at"] + updated = row["updated_at"] + return { + "id": row["id"], + "parentId": row["parent_id"], + "type": row["type"], + "name": row["name"], + "mimeType": row.get("mime_type") or "", + "storageKind": row.get("storage_kind") or "text", + "size": int(row.get("size") or 0), + "content": row.get("content_text") or "", + "previewText": row.get("preview_text") or "", + "isTruncatedPreview": bool(row.get("is_truncated_preview")), + "createdAt": int(created.timestamp() * 1000), + "updatedAt": int(updated.timestamp() * 1000), + } + + def _fetch_node(self, node_id: str) -> dict[str, Any]: + self._ensure_initialized() + with self._connect() as conn: + with conn.cursor() as cur: + cur.execute("SELECT * FROM document_nodes WHERE id = %s", (node_id,)) + row = cur.fetchone() + if row is None: + raise KeyError(node_id) + return row + + def list_nodes(self) -> list[dict[str, Any]]: + self._ensure_initialized() + with self._connect() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + id, + parent_id, + type, + name, + mime_type, + storage_kind, + size, + CASE + WHEN storage_kind = 'text' AND size <= %s THEN content_text + ELSE '' + END AS content_text, + preview_text, + is_truncated_preview, + created_at, + updated_at + FROM document_nodes + ORDER BY + CASE WHEN type = 'folder' THEN 0 ELSE 1 END, + LOWER(name), + created_at + """, + (MAX_TEXT_SIZE,), + ) + return [self._serialize(row) for row in cur.fetchall()] + + def create_folder(self, name: str, parent_id: str | None) -> dict[str, Any]: + self._ensure_initialized() + node_id = str(uuid.uuid4()) + with self._connect() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO document_nodes (id, parent_id, type, name, updated_at) + VALUES (%s, %s, 'folder', %s, CURRENT_TIMESTAMP) + RETURNING * + """, + (node_id, parent_id, name), + ) + return self._serialize(cur.fetchone()) + + def create_text_file(self, name: str, parent_id: str | None, content: str = "") -> dict[str, Any]: + self._ensure_initialized() + node_id = str(uuid.uuid4()) + payload = _text_payload(content) + with self._connect() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO document_nodes ( + id, parent_id, type, name, mime_type, storage_kind, size, + content_text, preview_text, is_truncated_preview, blob_data, updated_at + ) + VALUES (%s, %s, 'file', %s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP) + RETURNING * + """, + ( + node_id, + parent_id, + name, + infer_mime_type(name), + payload["storage_kind"], + payload["size"], + payload["content_text"], + payload["preview_text"], + payload["is_truncated_preview"], + None, + ), + ) + return self._serialize(cur.fetchone()) + + def upload_file(self, name: str, parent_id: str | None, raw_bytes: bytes, mime_type: str = "") -> dict[str, Any]: + self._ensure_initialized() + node_id = str(uuid.uuid4()) + payload = prepare_file_payload(name, raw_bytes, mime_type) + with self._connect() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO document_nodes ( + id, parent_id, type, name, mime_type, storage_kind, size, + content_text, preview_text, is_truncated_preview, blob_data, updated_at + ) + VALUES (%s, %s, 'file', %s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP) + RETURNING * + """, + ( + node_id, + parent_id, + name, + payload["mime_type"], + payload["storage_kind"], + payload["size"], + payload["content_text"], + payload["preview_text"], + payload["is_truncated_preview"], + payload["blob_data"], + ), + ) + return self._serialize(cur.fetchone()) + + def update_node(self, node_id: str, *, name: str | None | object = BaseDocumentStore._UNSET, parent_id: str | None | object = BaseDocumentStore._UNSET, content: str | None | object = BaseDocumentStore._UNSET) -> dict[str, Any]: + self._ensure_initialized() + row = self._fetch_node(node_id) + next_name = row["name"] if name is BaseDocumentStore._UNSET else name + next_parent_id = row["parent_id"] if parent_id is BaseDocumentStore._UNSET else parent_id + next_mime_type = row.get("mime_type") or "" + next_storage_kind = row.get("storage_kind") or "text" + next_size = row.get("size") or 0 + next_content_text = row.get("content_text") or "" + next_preview_text = row.get("preview_text") or "" + next_is_truncated = bool(row.get("is_truncated_preview")) + next_blob_data = row.get("blob_data") + + if row["type"] == "file" and name is not BaseDocumentStore._UNSET: + next_mime_type = infer_mime_type(next_name, next_mime_type) + if content is not BaseDocumentStore._UNSET: + payload = _text_payload(content) + next_storage_kind = payload["storage_kind"] + next_size = payload["size"] + next_content_text = payload["content_text"] + next_preview_text = payload["preview_text"] + next_is_truncated = payload["is_truncated_preview"] + next_blob_data = None + + with self._connect() as conn: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE document_nodes + SET + name = %s, + parent_id = %s, + mime_type = %s, + storage_kind = %s, + size = %s, + content_text = %s, + preview_text = %s, + is_truncated_preview = %s, + blob_data = %s, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + RETURNING * + """, + ( + next_name, + next_parent_id, + next_mime_type, + next_storage_kind, + next_size, + next_content_text, + next_preview_text, + next_is_truncated, + next_blob_data, + node_id, + ), + ) + updated = cur.fetchone() + if updated is None: + raise KeyError(node_id) + return self._serialize(updated) + + def replace_blob(self, node_id: str, filename: str, raw_bytes: bytes, mime_type: str = "") -> dict[str, Any]: + self._ensure_initialized() + payload = prepare_file_payload(filename, raw_bytes, mime_type) + with self._connect() as conn: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE document_nodes + SET + name = %s, + mime_type = %s, + storage_kind = %s, + size = %s, + content_text = %s, + preview_text = %s, + is_truncated_preview = %s, + blob_data = %s, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s AND type = 'file' + RETURNING * + """, + ( + filename, + payload["mime_type"], + payload["storage_kind"], + payload["size"], + payload["content_text"], + payload["preview_text"], + payload["is_truncated_preview"], + payload["blob_data"], + node_id, + ), + ) + updated = cur.fetchone() + if updated is None: + raise KeyError(node_id) + return self._serialize(updated) + + def delete_node(self, node_id: str) -> None: + self._ensure_initialized() + with self._connect() as conn: + with conn.cursor() as cur: + cur.execute( + """ + WITH RECURSIVE descendants AS ( + SELECT id FROM document_nodes WHERE id = %s + UNION ALL + SELECT child.id + FROM document_nodes child + INNER JOIN descendants parent ON child.parent_id = parent.id + ) + DELETE FROM document_nodes + WHERE id IN (SELECT id FROM descendants) + """, + (node_id,), + ) + + def get_blob(self, node_id: str) -> BlobPayload: + self._ensure_initialized() + with self._connect() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT id, name, mime_type, content_text, blob_data + FROM document_nodes + WHERE id = %s AND type = 'file' + """, + (node_id,), + ) + row = cur.fetchone() + if row is None: + raise FileNotFoundError(node_id) + blob_data = row.get("blob_data") + if blob_data is None: + content = (row.get("content_text") or "").encode("utf-8") + else: + content = bytes(blob_data) + return BlobPayload( + content=content, + mime_type=row.get("mime_type") or infer_mime_type(row.get("name") or ""), + filename=row["name"], + ) + + +_document_store: BaseDocumentStore | None = None + + +def get_document_store() -> BaseDocumentStore: + global _document_store + if _document_store is not None: + return _document_store + backend = (os.getenv("DOCS_BACKEND") or "postgres").strip().lower() + if backend == "memory": + _document_store = InMemoryDocumentStore() + return _document_store + + database_url = os.getenv("DATABASE_URL", "").strip() + if not database_url: + raise RuntimeError("缺少 DATABASE_URL，无法初始化 PostgreSQL 文档存储") + _document_store = PostgresDocumentStore(database_url) + return _document_store + + +def reset_document_store() -> None: + global _document_store + _document_store = None diff --git a/backend/main.py b/backend/main.py index 0b64d0c..a16156d 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,3 +1,4 @@ +import asyncio import base64 import json import logging @@ -5,12 +6,13 @@ import os import uuid from typing import Optional -from fastapi import FastAPI, HTTPException, Request, Security +from fastapi import FastAPI, File, Form, HTTPException, Request, Response, Security, UploadFile from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, StreamingResponse from fastapi.security import APIKeyHeader from pydantic import BaseModel +from docs_store import get_document_store from geoip import get_ip_location_text from job_handlers import ( _sanitize_converted_markdown, @@ -110,6 +112,23 @@ class ASRJobRequest(BaseModel): language: Optional[str] = "zh-CN" +class CreateFolderRequest(BaseModel): + name: str + parentId: Optional[str] = None + + +class CreateTextFileRequest(BaseModel): + name: str + parentId: Optional[str] = None + content: str = "" + + +class UpdateNodeRequest(BaseModel): + name: Optional[str] = None + parentId: Optional[str] = None + content: Optional[str] = None + + def _preview(text: str, limit: int = 80) -> str: value = (text or "").replace("\n", "\\n") if len(value) <= limit: @@ -219,6 +238,12 @@ async def _queue_load_snapshot() -> dict: return {} +async def _docs_store_call(method_name: str, *args, **kwargs): + store = get_document_store() + method = getattr(store, method_name) + return await asyncio.to_thread(method, *args, **kwargs) + + @app.post("/v1/completions") async def create_completion( request: Request, @@ -452,6 +477,127 @@ async def get_job_load(api_key: str = Security(get_api_key)): return {"queues": await _queue_load_snapshot()} +@app.get("/v1/docs/nodes") +async def list_docs_nodes(api_key: str = Security(get_api_key)): + del api_key + try: + return {"nodes": await _docs_store_call("list_nodes")} + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + + +@app.post("/v1/docs/folders") +async def create_docs_folder(req: CreateFolderRequest, api_key: str = Security(get_api_key)): + del api_key + if not (req.name or "").strip(): + raise HTTPException(status_code=400, detail="文件夹名称不能为空") + try: + node = await _docs_store_call("create_folder", req.name.strip(), req.parentId) + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + return {"node": node} + + +@app.post("/v1/docs/files/text") +async def create_docs_text_file(req: CreateTextFileRequest, api_key: str = Security(get_api_key)): + del api_key + if not (req.name or "").strip(): + raise HTTPException(status_code=400, detail="文件名称不能为空") + try: + node = await _docs_store_call("create_text_file", req.name.strip(), req.parentId, req.content or "") + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + return {"node": node} + + +@app.post("/v1/docs/files/upload") +async def upload_docs_file( + file: UploadFile = File(...), + parent_id: Optional[str] = Form(default=None), + api_key: str = Security(get_api_key), +): + del api_key + filename = (file.filename or "").strip() + if not filename: + raise HTTPException(status_code=400, detail="文件名称不能为空") + raw_bytes = await file.read() + try: + node = await _docs_store_call("upload_file", filename, parent_id, raw_bytes, file.content_type or "") + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + return {"node": node} + + +@app.patch("/v1/docs/nodes/{node_id}") +async def update_docs_node(node_id: str, req: UpdateNodeRequest, api_key: str = Security(get_api_key)): + del api_key + fields_set = req.model_fields_set if hasattr(req, "model_fields_set") else getattr(req, "__fields_set__", set()) + if not fields_set: + raise HTTPException(status_code=400, detail="缺少更新内容") + update_kwargs = {} + if "name" in fields_set: + next_name = req.name.strip() if isinstance(req.name, str) else "" + if not next_name: + raise HTTPException(status_code=400, detail="名称不能为空") + update_kwargs["name"] = next_name + if "parentId" in fields_set: + update_kwargs["parent_id"] = req.parentId + if "content" in fields_set: + update_kwargs["content"] = req.content or "" + try: + node = await _docs_store_call("update_node", node_id, **update_kwargs) + except KeyError as exc: + raise HTTPException(status_code=404, detail="节点不存在") from exc + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + return {"node": node} + + +@app.put("/v1/docs/files/{node_id}/blob") +async def replace_docs_blob( + node_id: str, + file: UploadFile = File(...), + api_key: str = Security(get_api_key), +): + del api_key + filename = (file.filename or "").strip() + if not filename: + raise HTTPException(status_code=400, detail="文件名称不能为空") + raw_bytes = await file.read() + try: + node = await _docs_store_call("replace_blob", node_id, filename, raw_bytes, file.content_type or "") + except KeyError as exc: + raise HTTPException(status_code=404, detail="节点不存在") from exc + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + return {"node": node} + + +@app.delete("/v1/docs/nodes/{node_id}") +async def delete_docs_node(node_id: str, api_key: str = Security(get_api_key)): + del api_key + try: + await _docs_store_call("delete_node", node_id) + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + return {"ok": True} + + +@app.get("/v1/docs/files/{node_id}/blob") +async def download_docs_blob(node_id: str, api_key: str = Security(get_api_key)): + del api_key + try: + payload = await _docs_store_call("get_blob", node_id) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail="文件不存在") from exc + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + headers = { + "Content-Disposition": f'inline; filename="{payload.filename}"', + } + return Response(content=payload.content, media_type=payload.mime_type, headers=headers) + + def _register_tts_asr_routes(): try: from tts_asr import register_tts_asr_routes diff --git a/backend/requirements.docker.txt b/backend/requirements.docker.txt new file mode 100644 index 0000000..4d98848 --- /dev/null +++ b/backend/requirements.docker.txt @@ -0,0 +1,10 @@ +fastapi>=0.95.0 +uvicorn[standard]>=0.23.0 +pydantic>=1.10.0 +httpx>=0.24.0 +redis>=5.0.0 +psycopg[binary]>=3.2.0 +python-multipart>=0.0.9 +python-dotenv>=1.0.0 +markitdown>=0.1.1 +geoip2>=4.8.0 diff --git a/backend/requirements.txt b/backend/requirements.txt index 4f867ba..82f8e5f 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -3,6 +3,8 @@ uvicorn[standard]>=0.23.0 pydantic>=1.10.0 httpx>=0.24.0 redis>=5.0.0 +psycopg[binary]>=3.2.0 +python-multipart>=0.0.9 python-dotenv>=1.0.0 numpy>=1.23.0 diff --git a/backend/tests/test_main_endpoints.py b/backend/tests/test_main_endpoints.py index e7c2f6f..8d4ec1e 100644 --- a/backend/tests/test_main_endpoints.py +++ b/backend/tests/test_main_endpoints.py @@ -7,6 +7,7 @@ from pathlib import Path from fastapi.testclient import TestClient os.environ["JOB_BACKEND"] = "memory" +os.environ["DOCS_BACKEND"] = "memory" CURRENT_DIR = Path(__file__).resolve().parent BACKEND_DIR = CURRENT_DIR.parent @@ -15,6 +16,7 @@ if str(BACKEND_DIR) not in sys.path: import job_handlers # type: ignore import job_system # type: ignore +import docs_store # type: ignore main = importlib.import_module("main") @@ -23,6 +25,7 @@ HEADERS = {"X-API-Key": main.API_KEY} def setup_function(): job_system.reset_job_manager() + docs_store.reset_document_store() main._handlers_registered = False @@ -119,3 +122,69 @@ def test_post_convert_unsupported_extension_returns_500(): }) assert resp.status_code == 500 assert "仅支持" in resp.json()["error"] + + +def test_docs_nodes_crud_round_trip(): + with TestClient(main.app) as client: + folder_resp = client.post("/v1/docs/folders", headers=HEADERS, json={ + "name": "项目资料", + "parentId": None, + }) + assert folder_resp.status_code == 200 + folder = folder_resp.json()["node"] + + file_resp = client.post("/v1/docs/files/text", headers=HEADERS, json={ + "name": "notes.md", + "parentId": folder["id"], + "content": "# hello", + }) + assert file_resp.status_code == 200 + file_node = file_resp.json()["node"] + assert file_node["previewText"] == "# hello" + + list_resp = client.get("/v1/docs/nodes", headers=HEADERS) + assert list_resp.status_code == 200 + nodes = list_resp.json()["nodes"] + assert len(nodes) == 2 + + rename_resp = client.patch(f"/v1/docs/nodes/{file_node['id']}", headers=HEADERS, json={ + "name": "renamed.md", + }) + assert rename_resp.status_code == 200 + assert rename_resp.json()["node"]["name"] == "renamed.md" + + blob_resp = client.get(f"/v1/docs/files/{file_node['id']}/blob", headers=HEADERS) + assert blob_resp.status_code == 200 + assert blob_resp.content == b"# hello" + + delete_resp = client.delete(f"/v1/docs/nodes/{folder['id']}", headers=HEADERS) + assert delete_resp.status_code == 200 + + final_list = client.get("/v1/docs/nodes", headers=HEADERS) + assert final_list.status_code == 200 + assert final_list.json()["nodes"] == [] + + +def test_docs_file_upload_and_blob_replace(): + with TestClient(main.app) as client: + upload_resp = client.post( + "/v1/docs/files/upload", + headers=HEADERS, + files={"file": ("image.png", b"png-bytes", "image/png")}, + data={"parent_id": ""}, + ) + assert upload_resp.status_code == 200 + node = upload_resp.json()["node"] + assert node["storageKind"] == "blob" + + replace_resp = client.put( + f"/v1/docs/files/{node['id']}/blob", + headers=HEADERS, + files={"file": ("photo.jpg", b"jpeg-bytes", "image/jpeg")}, + ) + assert replace_resp.status_code == 200 + assert replace_resp.json()["node"]["name"] == "photo.jpg" + + blob_resp = client.get(f"/v1/docs/files/{node['id']}/blob", headers=HEADERS) + assert blob_resp.status_code == 200 + assert blob_resp.content == b"jpeg-bytes" diff --git a/docker-compose.yml b/docker-compose.yml index 8b39e87..9362508 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,48 +1,67 @@ -version: "3.9" - services: - redis: - image: redis:7-alpine + frontend: + build: + context: . + dockerfile: Dockerfile.frontend + args: + DOCKER_REGISTRY_PREFIX: ${DOCKER_REGISTRY_PREFIX:-} + depends_on: + - api ports: - - "6379:6379" + - "8080:80" + + postgres: + image: ${DOCKER_REGISTRY_PREFIX:-}postgres:16-alpine + environment: + POSTGRES_DB: ${POSTGRES_DB:-llm_in_text} + POSTGRES_USER: ${POSTGRES_USER:-llm_in_text} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-llm_in_text_change_me} + volumes: + - ./docker-data/postgres:/var/lib/postgresql/data + + redis: + image: ${DOCKER_REGISTRY_PREFIX:-}redis:7-alpine command: ["redis-server", "--appendonly", "yes"] volumes: - - redis-data:/data + - ./docker-data/redis:/data api: - image: python:3.11-slim - working_dir: /app - command: sh -c "pip install -r backend/requirements.txt && python backend/main.py" + build: + context: . + dockerfile: backend/Dockerfile + args: + DOCKER_REGISTRY_PREFIX: ${DOCKER_REGISTRY_PREFIX:-} env_file: - backend/.env environment: JOB_BACKEND: redis REDIS_URL: redis://redis:6379/0 + DATABASE_URL: ${DATABASE_URL:-postgresql://llm_in_text:llm_in_text_change_me@postgres:5432/llm_in_text} + DOCS_BACKEND: postgres JOB_SHARED_TEMP_DIR: /shared-jobs - volumes: - - .:/app - - job-shared:/shared-jobs depends_on: + - postgres - redis - ports: - - "8001:8001" + volumes: + - ./docker-data/jobs:/shared-jobs worker: - image: python:3.11-slim - working_dir: /app - command: sh -c "pip install -r backend/requirements.txt && python backend/worker.py" + build: + context: . + dockerfile: backend/Dockerfile + args: + DOCKER_REGISTRY_PREFIX: ${DOCKER_REGISTRY_PREFIX:-} + command: ["python", "worker.py"] env_file: - backend/.env environment: JOB_BACKEND: redis REDIS_URL: redis://redis:6379/0 + DATABASE_URL: ${DATABASE_URL:-postgresql://llm_in_text:llm_in_text_change_me@postgres:5432/llm_in_text} + DOCS_BACKEND: postgres JOB_SHARED_TEMP_DIR: /shared-jobs - volumes: - - .:/app - - job-shared:/shared-jobs depends_on: + - postgres - redis - -volumes: - redis-data: - job-shared: + volumes: + - ./docker-data/jobs:/shared-jobs diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..87063fa --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,21 @@ +server { + listen 80; + server_name _; + + root /usr/share/nginx/html; + index index.html; + + location / { + try_files $uri $uri/ /index.html; + } + + location /v1/ { + proxy_pass http://api:8001/v1/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + } +} diff --git a/src/components/FileContent.vue b/src/components/FileContent.vue index a0d9a29..bb736c3 100644 --- a/src/components/FileContent.vue +++ b/src/components/FileContent.vue @@ -35,6 +35,8 @@ const imageEditorError = ref('') const isEditingImage = ref(false) const isSavingImage = ref(false) const videoPreviewError = ref('') +const resolvedFileBlob = ref(null) +let blobRequestToken = 0 const isRoot = computed(() => !props.node) const isFolder = computed(() => props.node?.type === 'folder') @@ -74,7 +76,7 @@ const folderItems = computed(() => { if (!isRoot.value && !isFolder.value) return [] return isRoot.value ? props.rootNodes : props.node.children || [] }) -const fileBlob = computed(() => props.getFileBlob(props.node)) +const fileBlob = computed(() => resolvedFileBlob.value) const previewText = computed(() => props.node?.content || props.node?.previewText || '') const lineCount = computed(() => { if (!previewText.value) return 0 @@ -105,8 +107,8 @@ function assignBasePreviewUrl(url = '') { } watch( - [fileBlob, () => props.node?.id], - ([blob]) => { + () => [props.node?.id, resolvedFileBlob.value], + ([, blob]) => { clearBasePreview() videoPreviewError.value = '' @@ -117,6 +119,25 @@ watch( { immediate: true } ) +watch( + () => props.node, + async (node) => { + const requestToken = ++blobRequestToken + resolvedFileBlob.value = null + if (!node || node.type !== 'file') return + + try { + const result = await props.getFileBlob(node) + if (requestToken !== blobRequestToken) return + resolvedFileBlob.value = result instanceof Blob ? result : null + } catch { + if (requestToken !== blobRequestToken) return + resolvedFileBlob.value = null + } + }, + { immediate: true } +) + watch( () => props.node?.id, () => { @@ -127,6 +148,7 @@ watch( ) onBeforeUnmount(() => { + blobRequestToken += 1 clearBasePreview() }) @@ -253,8 +275,8 @@ function downloadFile() {
-

{{ isRoot ? '本地文件空间' : node.name }}

-

{{ isRoot ? '所有文件都保存在当前浏览器本地，不会上传到服务器。' : '像 GitHub 一样浏览当前目录内容。' }}

+

{{ isRoot ? '文档空间' : node.name }}

+

{{ isRoot ? '所有文件都保存在当前服务端文档空间中。' : '像 GitHub 一样浏览当前目录内容。' }}

{{ folderItems.length }} 项 @@ -387,7 +409,7 @@ function downloadFile() {

视频预览暂时不可用

{{ videoPreviewError }}

-

原始文件仍保存在浏览器本地，你可以继续下载后使用本地播放器打开。

+

原始文件仍保存在服务端文档空间中，你可以继续下载后使用本地播放器打开。