mirror of
https://github.com/infiniflow/ragflow.git
synced 2025-06-26 22:19:57 +00:00

### What problem does this PR solve? This PR adds the support for latest OpenSearch2.19.1 as the store engine & search engine option for RAGFlow. ### Main Benefit 1. OpenSearch2.19.1 is licensed under the [Apache v2.0 License] which is much better than Elasticsearch 2. For search, OpenSearch2.19.1 supports full-text search、vector_search、hybrid_search those are similar with Elasticsearch on schema 3. For store, OpenSearch2.19.1 stores text、vector those are quite simliar with Elasticsearch on schema ### Changes - Support opensearch_python_connetor. I make a lot of adaptions since the schema and api/method between ES and Opensearch differs in many ways(especially the knn_search has a significant gap) : rag/utils/opensearch_coon.py - Support static config adaptions by changing: conf/service_conf.yaml、api/settings.py、rag/settings.py - Supprt some store&search schema changes between OpenSearch and ES: conf/os_mapping.json - Support OpenSearch python sdk : pyproject.toml - Support docker config for OpenSearch2.19.1 : docker/.env、docker/docker-compose-base.yml、docker/service_conf.yaml.template ### How to use - I didn't change the priority that ES as the default doc/search engine. Only if in docker/.env , we set DOC_ENGINE=${DOC_ENGINE:-opensearch}, it will work. ### Others Our team tested a lot of docs in our environment by using OpenSearch as the vector database ,it works very well. All the conifg for OpenSearch is necessary. ### Type of change - [x] New Feature (non-breaking change which adds functionality) --------- Co-authored-by: Yongteng Lei <yongtengrey@outlook.com> Co-authored-by: writinwaters <93570324+writinwaters@users.noreply.github.com> Co-authored-by: Yingfeng <yingfeng.zhang@gmail.com>
188 lines
4.8 KiB
YAML
188 lines
4.8 KiB
YAML
services:
|
|
es01:
|
|
container_name: ragflow-es-01
|
|
profiles:
|
|
- elasticsearch
|
|
image: elasticsearch:${STACK_VERSION}
|
|
volumes:
|
|
- esdata01:/usr/share/elasticsearch/data
|
|
ports:
|
|
- ${ES_PORT}:9200
|
|
env_file: .env
|
|
environment:
|
|
- node.name=es01
|
|
- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
|
|
- bootstrap.memory_lock=false
|
|
- discovery.type=single-node
|
|
- xpack.security.enabled=true
|
|
- xpack.security.http.ssl.enabled=false
|
|
- xpack.security.transport.ssl.enabled=false
|
|
- cluster.routing.allocation.disk.watermark.low=5gb
|
|
- cluster.routing.allocation.disk.watermark.high=3gb
|
|
- cluster.routing.allocation.disk.watermark.flood_stage=2gb
|
|
- TZ=${TIMEZONE}
|
|
mem_limit: ${MEM_LIMIT}
|
|
ulimits:
|
|
memlock:
|
|
soft: -1
|
|
hard: -1
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl http://localhost:9200"]
|
|
interval: 10s
|
|
timeout: 10s
|
|
retries: 120
|
|
networks:
|
|
- ragflow
|
|
restart: on-failure
|
|
|
|
opensearch01:
|
|
container_name: ragflow-opensearch-01
|
|
profiles:
|
|
- opensearch
|
|
image: hub.icert.top/opensearchproject/opensearch:2.19.1
|
|
volumes:
|
|
- osdata01:/usr/share/opensearch/data
|
|
ports:
|
|
- ${OS_PORT}:9201
|
|
env_file: .env
|
|
environment:
|
|
- node.name=opensearch01
|
|
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
|
|
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}
|
|
- bootstrap.memory_lock=false
|
|
- discovery.type=single-node
|
|
- plugins.security.disabled=false
|
|
- plugins.security.ssl.http.enabled=false
|
|
- plugins.security.ssl.transport.enabled=true
|
|
- cluster.routing.allocation.disk.watermark.low=5gb
|
|
- cluster.routing.allocation.disk.watermark.high=3gb
|
|
- cluster.routing.allocation.disk.watermark.flood_stage=2gb
|
|
- TZ=${TIMEZONE}
|
|
- http.port=9201
|
|
mem_limit: ${MEM_LIMIT}
|
|
ulimits:
|
|
memlock:
|
|
soft: -1
|
|
hard: -1
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl http://localhost:9201"]
|
|
interval: 10s
|
|
timeout: 10s
|
|
retries: 120
|
|
networks:
|
|
- ragflow
|
|
restart: on-failure
|
|
|
|
infinity:
|
|
container_name: ragflow-infinity
|
|
profiles:
|
|
- infinity
|
|
image: infiniflow/infinity:v0.6.0-dev3
|
|
volumes:
|
|
- infinity_data:/var/infinity
|
|
- ./infinity_conf.toml:/infinity_conf.toml
|
|
command: ["-f", "/infinity_conf.toml"]
|
|
ports:
|
|
- ${INFINITY_THRIFT_PORT}:23817
|
|
- ${INFINITY_HTTP_PORT}:23820
|
|
- ${INFINITY_PSQL_PORT}:5432
|
|
env_file: .env
|
|
environment:
|
|
- TZ=${TIMEZONE}
|
|
mem_limit: ${MEM_LIMIT}
|
|
ulimits:
|
|
nofile:
|
|
soft: 500000
|
|
hard: 500000
|
|
networks:
|
|
- ragflow
|
|
healthcheck:
|
|
test: ["CMD", "curl", "http://localhost:23820/admin/node/current"]
|
|
interval: 10s
|
|
timeout: 10s
|
|
retries: 120
|
|
restart: on-failure
|
|
|
|
|
|
mysql:
|
|
# mysql:5.7 linux/arm64 image is unavailable.
|
|
image: mysql:8.0.39
|
|
container_name: ragflow-mysql
|
|
env_file: .env
|
|
environment:
|
|
- MYSQL_ROOT_PASSWORD=${MYSQL_PASSWORD}
|
|
- TZ=${TIMEZONE}
|
|
command:
|
|
--max_connections=1000
|
|
--character-set-server=utf8mb4
|
|
--collation-server=utf8mb4_unicode_ci
|
|
--default-authentication-plugin=mysql_native_password
|
|
--tls_version="TLSv1.2,TLSv1.3"
|
|
--init-file /data/application/init.sql
|
|
--binlog_expire_logs_seconds=604800
|
|
ports:
|
|
- ${MYSQL_PORT}:3306
|
|
volumes:
|
|
- mysql_data:/var/lib/mysql
|
|
- ./init.sql:/data/application/init.sql
|
|
networks:
|
|
- ragflow
|
|
healthcheck:
|
|
test: ["CMD", "mysqladmin" ,"ping", "-uroot", "-p${MYSQL_PASSWORD}"]
|
|
interval: 10s
|
|
timeout: 10s
|
|
retries: 3
|
|
restart: on-failure
|
|
|
|
minio:
|
|
image: quay.io/minio/minio:RELEASE.2023-12-20T01-00-02Z
|
|
container_name: ragflow-minio
|
|
command: server --console-address ":9001" /data
|
|
ports:
|
|
- ${MINIO_PORT}:9000
|
|
- ${MINIO_CONSOLE_PORT}:9001
|
|
env_file: .env
|
|
environment:
|
|
- MINIO_ROOT_USER=${MINIO_USER}
|
|
- MINIO_ROOT_PASSWORD=${MINIO_PASSWORD}
|
|
- TZ=${TIMEZONE}
|
|
volumes:
|
|
- minio_data:/data
|
|
networks:
|
|
- ragflow
|
|
restart: on-failure
|
|
|
|
redis:
|
|
# swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/valkey/valkey:8
|
|
image: valkey/valkey:8
|
|
container_name: ragflow-redis
|
|
command: redis-server --requirepass ${REDIS_PASSWORD} --maxmemory 128mb --maxmemory-policy allkeys-lru
|
|
env_file: .env
|
|
ports:
|
|
- ${REDIS_PORT}:6379
|
|
volumes:
|
|
- redis_data:/data
|
|
networks:
|
|
- ragflow
|
|
restart: on-failure
|
|
|
|
|
|
|
|
volumes:
|
|
esdata01:
|
|
driver: local
|
|
osdata01:
|
|
driver: local
|
|
infinity_data:
|
|
driver: local
|
|
mysql_data:
|
|
driver: local
|
|
minio_data:
|
|
driver: local
|
|
redis_data:
|
|
driver: local
|
|
|
|
networks:
|
|
ragflow:
|
|
driver: bridge
|