feat: add web search block functionality and integrate with existing plugins

- Introduced a new web search block plugin to handle web search queries and results.
- Updated copilot, doc block, and pro block plugins to include web search context in AI completions.
- Implemented utility functions for parsing and building web search markdown.
- Enhanced API to support web search requests and responses.
- Added configuration for web search URL and timeout settings.
- Updated size limit checks to account for web search content.

docker-compose.yml

@@ -25,6 +25,61 @@ services:
     volumes:
       - ./docker-data/redis:/data
 
+  searxng:
+    image: ${DOCKER_REGISTRY_PREFIX:-}searxng/searxng:latest
+    restart: unless-stopped
+    environment:
+      BASE_URL: http://searxng:8080/
+      INSTANCE_NAME: llm-in-text-search
+    volumes:
+      - ./docker-data/searxng:/etc/searxng
+
+  firecrawl-postgres:
+    image: ${DOCKER_REGISTRY_PREFIX:-}postgres:16-alpine
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: firecrawl
+      POSTGRES_USER: firecrawl
+      POSTGRES_PASSWORD: firecrawl
+    volumes:
+      - ./docker-data/firecrawl-postgres:/var/lib/postgresql/data
+
+  firecrawl-rabbitmq:
+    image: ${DOCKER_REGISTRY_PREFIX:-}rabbitmq:3-management-alpine
+    restart: unless-stopped
+    volumes:
+      - ./docker-data/firecrawl-rabbitmq:/var/lib/rabbitmq
+
+  firecrawl-playwright:
+    image: ghcr.io/firecrawl/playwright-service:latest
+    restart: unless-stopped
+    environment:
+      PORT: 3000
+
+  firecrawl:
+    image: ghcr.io/firecrawl/firecrawl:latest
+    restart: unless-stopped
+    environment:
+      HOST: 0.0.0.0
+      PORT: 3002
+      POSTGRES_HOST: firecrawl-postgres
+      POSTGRES_PORT: 5432
+      POSTGRES_DB: firecrawl
+      POSTGRES_USER: firecrawl
+      POSTGRES_PASSWORD: firecrawl
+      REDIS_URL: redis://redis:6379/1
+      REDIS_RATE_LIMIT_URL: redis://redis:6379/1
+      NUQ_RABBITMQ_URL: amqp://guest:guest@firecrawl-rabbitmq:5672/
+      PLAYWRIGHT_MICROSERVICE_URL: http://firecrawl-playwright:3000/scrape
+      SEARXNG_ENDPOINT: http://searxng:8080
+      USE_DB_AUTHENTICATION: "false"
+      FIRECRAWL_API_KEY: ${FIRECRAWL_API_KEY:-change-me}
+    depends_on:
+      - redis
+      - firecrawl-postgres
+      - firecrawl-rabbitmq
+      - firecrawl-playwright
+
   api:
     build:
       context: .
@@ -39,9 +94,13 @@ services:
       DATABASE_URL: ${DATABASE_URL:-postgresql://llm_in_text:llm_in_text_change_me@postgres:5432/llm_in_text}
       DOCS_BACKEND: postgres
       JOB_SHARED_TEMP_DIR: /shared-jobs
+      SEARXNG_BASE_URL: http://searxng:8080
+      FIRECRAWL_BASE_URL: http://firecrawl:3002
     depends_on:
       - postgres
       - redis
+      - searxng
+      - firecrawl
     ports:
       - "8001:8001"
     volumes:
@@ -62,8 +121,12 @@ services:
       DATABASE_URL: ${DATABASE_URL:-postgresql://llm_in_text:llm_in_text_change_me@postgres:5432/llm_in_text}
       DOCS_BACKEND: postgres
       JOB_SHARED_TEMP_DIR: /shared-jobs
+      SEARXNG_BASE_URL: http://searxng:8080
+      FIRECRAWL_BASE_URL: http://firecrawl:3002
     depends_on:
       - postgres
       - redis
+      - searxng
+      - firecrawl
     volumes:
       - ./docker-data/jobs:/shared-jobs