Merge pull request #1914 from danielaskdd/feat-tiktoken-cache

feat: add tiktoken cache directory support for offline deployment

.gitignore

@@ -50,6 +50,7 @@ rag_storage/
 examples/input/
 examples/output/
 output*/
+data/
 
 # Miscellaneous
 .DS_Store

docker-compose.yml

@@ -12,10 +12,13 @@ services:
     volumes:
       - ./data/rag_storage:/app/data/rag_storage
       - ./data/inputs:/app/data/inputs
+      - ./data/tiktoken:/app/data/tiktoken
       - ./config.ini:/app/config.ini
       - ./.env:/app/.env
     env_file:
       - .env
+    environment:
+      - TIKTOKEN_CACHE_DIR=/app/data/tiktoken
     restart: unless-stopped
     extra_hosts:
       - "host.docker.internal:host-gateway"

env.example

@@ -20,6 +20,9 @@ WEBUI_DESCRIPTION="Simple and Fast Graph Based RAG System"
 # INPUT_DIR=<absolute_path_for_doc_input_dir>
 # WORKING_DIR=<absolute_path_for_working_dir>
 
+### Tiktoken cache directory (Store cached files in this folder for offline deployment)
+# TIKTOKEN_CACHE_DIR=./temp/tiktoken
+
 ### Ollama Emulating Model and Tag
 # OLLAMA_EMULATING_MODEL_NAME=lightrag
 OLLAMA_EMULATING_MODEL_TAG=latest