feat: switch from OpenAI API to native Ollama Python client
This commit refactors the LLM integration to use Ollama's native Python client instead of OpenAI-compatible API, while fixing critical template syntax errors and improving project structure. Key changes: - Replace openai package with ollama package in backend requirements - Rewrite llm.py to use ollama.AsyncClient for direct Ollama API calls - Update main.py to use non-streaming Ollama responses with thinking extraction - Fix template syntax error in MilkdownEditor.vue (GhostTextOverlay component tags) - Fix string截取错误 by using slice() instead of substring() - Add src/utils/api.js and src/utils/config.js for shared configuration - Add CORS middleware to FastAPI backend - Update prompt.py with clearer instructions for continuation generation - Add comprehensive README.md documentation BREAKING CHANGE: Environment variables OLLAMA_BASE_URL changed to OLLAMA_HOST (remove /v1/ suffix)
This commit is contained in:
“ydy0615”
+1
-1
@@ -1,3 +1,3 @@
|
||||
OPENAI_API_KEY=ollama
|
||||
OLLAMA_BASE_URL=http://100.124.143.24:11434/v1/
|
||||
OLLAMA_HOST=http://192.168.0.120:11434
|
||||
OLLAMA_MODEL=gpt-oss:120b
|
||||
|
||||
Binary file not shown.
Binary file not shown.
+32
-49
@@ -1,69 +1,52 @@
|
||||
import os
|
||||
from typing import AsyncGenerator
|
||||
from openai import AsyncOpenAI
|
||||
import json
|
||||
import time
|
||||
import ollama
|
||||
from typing import AsyncGenerator
|
||||
|
||||
api_key = os.getenv('OPENAI_API_KEY', 'ollama')
|
||||
base_url = os.getenv('OLLAMA_BASE_URL', 'http://192.168.0.120:11434/v1/')
|
||||
model = os.getenv('OLLAMA_MODEL', 'gpt-oss:120b')
|
||||
OLLAMA_MODEL = os.getenv('OLLAMA_MODEL', 'gpt-oss:120b')
|
||||
OLLAMA_HOST = os.getenv('OLLAMA_BASE_URL', 'http://192.168.0.120:11434')
|
||||
|
||||
print(f"[LLM] API key configured: {'Yes' if api_key else 'No'}")
|
||||
print(f"[LLM] Base URL: {base_url}")
|
||||
print(f"[LLM] Model: {model}")
|
||||
# 移除 /v1/ 后缀（如果有的话），因为 Ollama Python 包使用原生 API
|
||||
if OLLAMA_HOST.endswith('/v1/'):
|
||||
OLLAMA_HOST = OLLAMA_HOST[:-4]
|
||||
elif OLLAMA_HOST.endswith('/v1'):
|
||||
OLLAMA_HOST = OLLAMA_HOST[:-3]
|
||||
|
||||
client = AsyncOpenAI(api_key=api_key, base_url=base_url)
|
||||
os.environ['OLLAMA_HOST'] = OLLAMA_HOST
|
||||
|
||||
print(f"[LLM] Ollama host: {OLLAMA_HOST}")
|
||||
print(f"[LLM] Model: {OLLAMA_MODEL}")
|
||||
|
||||
client = ollama.AsyncClient(host=OLLAMA_HOST)
|
||||
|
||||
async def stream_openai(prompt: str) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
调用 OpenAI/Ollama API 并流式返回补全内容。
|
||||
参考 completions-sample-code 的 streaming 逻辑。
|
||||
"""
|
||||
start_time = time.time()
|
||||
print(f"[LLM] ========== API Call Start ==========")
|
||||
print(f"[LLM] Prompt length: {len(prompt)}")
|
||||
print(f"[LLM] Model: {model}")
|
||||
print(f"[LLM] Calling Ollama API with prompt length: {len(prompt)}")
|
||||
|
||||
try:
|
||||
print(f"[LLM] Creating streaming chat completion...")
|
||||
stream = await client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
print(f"[LLM] Awaiting client.chat...")
|
||||
stream = await client.chat(
|
||||
model=OLLAMA_MODEL,
|
||||
messages=[{'role': 'user', 'content': prompt}],
|
||||
stream=True,
|
||||
max_tokens=128,
|
||||
temperature=0.2,
|
||||
options={
|
||||
'num_predict': 8192,
|
||||
'temperature': 0.2,
|
||||
}
|
||||
)
|
||||
|
||||
print(f"[LLM] Stream created successfully, iterating...")
|
||||
print(f"[LLM] Got stream object, starting iteration...")
|
||||
|
||||
chunk_count = 0
|
||||
first_chunk_time = None
|
||||
|
||||
async for chunk in stream:
|
||||
current_time = time.time()
|
||||
if first_chunk_time is None:
|
||||
first_chunk_time = current_time - start_time
|
||||
|
||||
chunk_count += 1
|
||||
choice = chunk.choices[0] if chunk.choices else None
|
||||
|
||||
if choice and choice.delta.content:
|
||||
content = choice.delta.content
|
||||
print(f"[LLM] Chunk {chunk_count}: '{content}' (latency: {current_time - start_time:.3f}s)")
|
||||
if chunk['message'] and chunk['message']['content']:
|
||||
content = chunk['message']['content']
|
||||
chunk_count += 1
|
||||
print(f"[LLM] Chunk {chunk_count}: {content}")
|
||||
yield json.dumps({"content": content})
|
||||
elif chunk.choices and hasattr(chunk.choices[0], 'finish_reason'):
|
||||
finish_reason = chunk.choices[0].finish_reason
|
||||
print(f"[LLM] Chunk {chunk_count}: finish_reason={finish_reason}")
|
||||
if finish_reason:
|
||||
break
|
||||
else:
|
||||
print(f"[LLM] Chunk {chunk_count}: empty or no content")
|
||||
|
||||
total_time = time.time() - start_time
|
||||
print(f"[LLM] Stream complete - chunks: {chunk_count}, first chunk latency: {first_chunk_time:.3f}s, total time: {total_time:.3f}s")
|
||||
print(f"[LLM] ========== API Call End ==========")
|
||||
print(f"[LLM] Stream complete, total chunks: {chunk_count}")
|
||||
except Exception as e:
|
||||
error_msg = f"Error: {str(e)}"
|
||||
print(f"[LLM] Error: {error_msg}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
yield json.dumps({"error": str(e), "type": type(e).__name__})
|
||||
yield json.dumps({"error": str(e)})
|
||||
|
||||
+121
-54
@@ -1,75 +1,142 @@
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import StreamingResponse, JSONResponse
|
||||
from pydantic import BaseModel
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import re
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
print("[Main] Backend service starting...")
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
class CompletionRequest(BaseModel):
|
||||
prefix: str
|
||||
suffix: str
|
||||
languageId: str = 'markdown'
|
||||
|
||||
def generate_stream(request: CompletionRequest):
|
||||
from prompt import build_prompt
|
||||
from llm import stream_openai
|
||||
def extract_completion_from_thinking(thinking: str) -> str:
|
||||
"""
|
||||
从模型的 thinking 输出中提取实际的续写内容。
|
||||
移除推理过程，保留实际的续写。
|
||||
"""
|
||||
if not thinking:
|
||||
return ""
|
||||
|
||||
start_time = time.time()
|
||||
print(f"[Main] ========== New Request ==========")
|
||||
print(f"[Main] prefix length: {len(request.prefix)}, suffix length: {len(request.suffix)}")
|
||||
print(f"[Main] languageId: {request.languageId}")
|
||||
print(f"[Main] Prefix (last 200 chars): '{request.prefix[-200:]}'")
|
||||
print(f"[Main] Suffix (first 200 chars): '{request.suffix[:200]}'")
|
||||
# 尝试找到实际的续写内容
|
||||
# 模型通常会在 thinking 中描述上下文，然后输出实际续写
|
||||
# 常见的模式是：推理过程以描述开始，然后直接输出续写
|
||||
|
||||
try:
|
||||
prompt = build_prompt(request.prefix, request.suffix)
|
||||
print(f"[Main] Built prompt length: {len(prompt)}")
|
||||
print(f"[Main] Prompt (first 300 chars): '{prompt[:300]}'")
|
||||
print(f"[Main] Prompt (last 200 chars): '{prompt[-200:]}'")
|
||||
|
||||
async def gen():
|
||||
chunk_count = 0
|
||||
first_chunk_time = None
|
||||
try:
|
||||
async for chunk in stream_openai(prompt):
|
||||
current_time = time.time()
|
||||
if first_chunk_time is None:
|
||||
first_chunk_time = current_time - start_time
|
||||
chunk_count += 1
|
||||
chunk_data = json.loads(chunk) if isinstance(chunk, str) else chunk
|
||||
content_preview = chunk_data.get('content', '')[:50] if chunk_data.get('content') else ''
|
||||
print(f"[Main] Chunk {chunk_count}: '{content_preview}'...")
|
||||
yield f"data: {json.dumps(chunk_data)}\n\n"
|
||||
|
||||
done_signal = {"done": True}
|
||||
total_time = time.time() - start_time
|
||||
print(f"[Main] Stream complete - total chunks: {chunk_count}, first chunk at: {first_chunk_time:.2f}s, total time: {total_time:.2f}s")
|
||||
yield f"data: {json.dumps(done_signal)}\n\n"
|
||||
except Exception as e:
|
||||
error_msg = {"error": str(e), "type": type(e).__name__}
|
||||
print(f"[Main] Generator error: {e}")
|
||||
yield f"data: {json.dumps(error_msg)}\n\n"
|
||||
return gen()
|
||||
except Exception as e:
|
||||
error_msg = {"error": str(e), "type": type(e).__name__}
|
||||
print(f"[Main] Error building prompt or calling LLM: {e}")
|
||||
yield f"data: {json.dumps(error_msg)}\n\n"
|
||||
# 查找 "Continuation:" 或类似标记之后的内容
|
||||
continuation_match = re.search(r'Continuation[:\s]*([\s\S]*)', thinking, re.IGNORECASE)
|
||||
if continuation_match:
|
||||
result = continuation_match.group(1).strip()
|
||||
# 移除可能的后续推理说明
|
||||
result = re.sub(r'\s*It seems like.*$', '', result, flags=re.IGNORECASE)
|
||||
return result.strip()
|
||||
|
||||
# 如果没有明确标记，尝试移除描述性内容
|
||||
# 查找 "We need to continue" 或类似开头
|
||||
continue_match = re.search(r'(?:We need to|Then we should|So we|I will|The|Thus)[,\s]+([A-Z][^.!?]*(?:[.!?]|$))', thinking)
|
||||
if continue_match:
|
||||
# 取找到的句子及其后续内容
|
||||
start_idx = continue_match.start(1)
|
||||
result = thinking[start_idx:].strip()
|
||||
# 移除 "Probably " 开头及其后续内容
|
||||
result = re.sub(r'^Probably\s+', '', result)
|
||||
# 如果有 "It seems like" 或类似短语，截断
|
||||
result = re.split(r'\s*It seems like\s', result, flags=re.IGNORECASE)[0]
|
||||
return result.strip()
|
||||
|
||||
# 最后的策略：直接返回 thinking，移除末尾的推理说明
|
||||
result = thinking.strip()
|
||||
# 移除 "Probably" 及其后续内容
|
||||
result = re.split(r'\s+Probably\s', result, flags=re.IGNORECASE, maxsplit=1)[0]
|
||||
# 移除 "The instruction:" 及其后续内容
|
||||
result = re.split(r'\s+The instruction:', result, flags=re.IGNORECASE, maxsplit=1)[0]
|
||||
|
||||
return result.strip()
|
||||
|
||||
@app.post("/v1/completions")
|
||||
async def create_completion(request: CompletionRequest):
|
||||
print(f"[Main] POST /v1/completions called at {time.time()}")
|
||||
return StreamingResponse(generate_stream(request), media_type="text/event-stream")
|
||||
|
||||
@app.get("/health")
|
||||
async def health_check():
|
||||
return {"status": "healthy", "timestamp": time.time()}
|
||||
from prompt import build_prompt
|
||||
import ollama
|
||||
|
||||
print(f"[Backend] POST /v1/completions called")
|
||||
print(f"[Backend] Received request - prefix length: {len(request.prefix)}, suffix length: {len(request.suffix)}")
|
||||
|
||||
OLLAMA_MODEL = os.getenv('OLLAMA_MODEL', 'gpt-oss:120b')
|
||||
OLLAMA_HOST = os.getenv('OLLAMA_HOST', 'http://192.168.0.120:11434')
|
||||
|
||||
print(f"[LLM] Using host: {OLLAMA_HOST}, model: {OLLAMA_MODEL}")
|
||||
|
||||
try:
|
||||
prompt = build_prompt(request.prefix, request.suffix)
|
||||
print(f"[Backend] Built prompt (first 100 chars): {prompt[:100]}...")
|
||||
print(f"[LLM] Full prompt:\n{prompt}\n")
|
||||
|
||||
# 使用非流式 API 获取完整响应
|
||||
print(f"[LLM] Calling Ollama API (non-streaming)...")
|
||||
client = ollama.AsyncClient(host=OLLAMA_HOST)
|
||||
response = await client.chat(
|
||||
model=OLLAMA_MODEL,
|
||||
messages=[{'role': 'user', 'content': prompt}],
|
||||
stream=False,
|
||||
options={
|
||||
'num_predict': 8192,
|
||||
'temperature': 0.2,
|
||||
}
|
||||
)
|
||||
|
||||
print(f"[LLM] Response type: {type(response)}")
|
||||
|
||||
# 提取 content 和 thinking
|
||||
content = ""
|
||||
thinking = ""
|
||||
|
||||
if hasattr(response, 'message') and response.message:
|
||||
content = response.message.content or ""
|
||||
thinking = getattr(response.message, 'thinking', '') or ""
|
||||
elif isinstance(response, dict):
|
||||
msg = response.get('message', {})
|
||||
content = msg.get('content', '') or ""
|
||||
thinking = msg.get('thinking', '') or ""
|
||||
|
||||
print(f"[LLM] Original content: {repr(content[:100] if content else '')}...")
|
||||
print(f"[LLM] Thinking length: {len(thinking)}")
|
||||
print(f"[LLM] Thinking (first 200): {thinking[:200]}...")
|
||||
|
||||
# 如果 content 为空，尝试从 thinking 中提取
|
||||
if not content and thinking:
|
||||
print(f"[LLM] Content is empty, extracting from thinking...")
|
||||
content = extract_completion_from_thinking(thinking)
|
||||
print(f"[LLM] Extracted completion: {repr(content[:100])}...")
|
||||
|
||||
print(f"[LLM] Final content length: {len(content)}")
|
||||
|
||||
# 返回完整内容
|
||||
async def generate():
|
||||
if content:
|
||||
print(f"[LLM] Yielding full content: {repr(content)}")
|
||||
yield f"data: {json.dumps({'content': content})}\n\n"
|
||||
yield f"data: {{'done': true}}\n\n"
|
||||
|
||||
return StreamingResponse(generate(), media_type="text/event-stream")
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"{{\"error\": \"{str(e)}\"}}"
|
||||
print(f"[Backend] Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return JSONResponse(content={"error": str(e)}, status_code=500)
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
port = int(os.getenv('PORT', 8000))
|
||||
print(f"[Main] Starting server on http://0.0.0.0:{port}")
|
||||
uvicorn.run(app, host="0.0.0.0", port=port)
|
||||
print("[Backend] Starting server on http://0.0.0.0:8000")
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
|
||||
+15
-9
@@ -3,8 +3,8 @@ from typing import Tuple
|
||||
|
||||
def build_prompt(prefix: str, suffix: str) -> str:
|
||||
"""
|
||||
构建用于代码补全的 Prompt。
|
||||
参考 completions-sample-code 的 extractPrompt 逻辑简化实现。
|
||||
改进后的提示词构建函数。
|
||||
使用更明确的指令来引导模型生成高质量的续写内容。
|
||||
"""
|
||||
MAX_CONTEXT_LINES = 30
|
||||
|
||||
@@ -14,15 +14,21 @@ def build_prompt(prefix: str, suffix: str) -> str:
|
||||
recent_prefix = '\n'.join(prefix_lines[-MAX_CONTEXT_LINES:])
|
||||
recent_suffix = '\n'.join(suffix_lines[:5])
|
||||
|
||||
prompt = f"""
|
||||
You are a helpful writing assistant. Continue the text naturally based on the context.
|
||||
prompt = f"""You are an expert writing assistant. Continue the text naturally.
|
||||
|
||||
Context (before cursor):
|
||||
{recent_prefix}
|
||||
CONTEXT:
|
||||
⟨CURSOR⟩ marks where to continue.
|
||||
- Before ⟨CURSOR⟩: existing text
|
||||
- After ⟨CURSOR⟩: following context (if any)
|
||||
|
||||
Complete this:
|
||||
{suffix if suffix else '(cursor here)'}
|
||||
RULES:
|
||||
- Match existing style, tone, and terminology
|
||||
- Maintain logical flow
|
||||
- Write only the continuation, nothing else
|
||||
|
||||
Continue:"""
|
||||
TEXT:
|
||||
{recent_prefix}⟨CURSOR⟩{recent_suffix}
|
||||
|
||||
CONTINUATION:"""
|
||||
|
||||
return prompt.strip()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
fastapi
|
||||
uvicorn
|
||||
openai
|
||||
ollama
|
||||
pydantic
|
||||
python-dotenv
|
||||
httpx
|
||||
|
||||
Reference in New Issue
Block a user