Update GPU docker & fix race condition with multiple workers (#436)

* fix gpu CMD and set tag to latest

* udpate dockerfiles. resolve race condition of index creation with multiple workers

* update dockerfiles for preload. remove try catch for elastic index creation

* add back try/catch. disable multiproc in default config to comply with --preload of gunicorn

* change to pip3 for GPU dockerfile

* remove --preload for gpu

Dockerfile

@@ -2,18 +2,20 @@ FROM python:3.7.4-stretch
 
 WORKDIR /home/user
 
+# copy code
+COPY haystack /home/user/haystack
+
 # install as a package
 COPY setup.py requirements.txt README.rst /home/user/
 RUN pip install -r requirements.txt
 RUN pip install -e .
 
-# copy code
-COPY haystack /home/user/haystack
-COPY rest_api /home/user/rest_api
-
-# copy saved FARM models
+# copy saved models
 COPY README.rst models* /home/user/models/
 
+# Copy REST API code
+COPY rest_api /home/user/rest_api
+
 # optional : copy sqlite db if needed for testing
 #COPY qa.db /home/user/
 
@@ -23,4 +25,4 @@ COPY README.rst models* /home/user/models/
 EXPOSE 8000
 
 # cmd for running the API
-CMD ["gunicorn", "rest_api.application:app",  "-b", "0.0.0.0", "-k", "uvicorn.workers.UvicornWorker", "--workers", "2", "--timeout", "180"]
+CMD ["gunicorn", "rest_api.application:app",  "-b", "0.0.0.0", "-k", "uvicorn.workers.UvicornWorker", "--workers", "1", "--timeout", "180", "--preload"]

Dockerfile-GPU

@@ -4,6 +4,9 @@ WORKDIR /home/user
 
 RUN apt-get update && apt-get install -y python3.7 python3.7-dev python3.7-distutils python3-pip curl git
 
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
 # Set default Python version
 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
 RUN update-alternatives --set python3 /usr/bin/python3.7
@@ -16,20 +19,19 @@ COPY setup.py requirements.txt README.rst /home/user/
 RUN pip3 install -r requirements.txt
 RUN pip3 install -e .
 
-# copy saved FARM models
+# copy saved models
 COPY README.rst models* /home/user/models/
 
-# Optional: copy sqlite db if needed for testing
+# Copy REST API code
+COPY rest_api /home/user/rest_api
+
+# optional : copy sqlite db if needed for testing
 #COPY qa.db /home/user/
 
-# Optional: copy data directory containing docs for indexing
+# optional: copy data directory containing docs for ingestion
 #COPY data /home/user/data
 
 EXPOSE 8000
 
-ENV LC_ALL=C.UTF-8
-ENV LANG=C.UTF-8
-
-
-# cmd for running the API
-CMD ["gunicorn", "haystack.api.application:app", "-b", "0.0.0.0", "-k", "uvicorn.workers.UvicornWorker", "--workers", "2", "--timeout", "180"]
+# cmd for running the API (note: "--preload" is not working with cuda)
+CMD ["gunicorn", "rest_api.application:app",  "-b", "0.0.0.0", "-k", "uvicorn.workers.UvicornWorker", "--workers", "1", "--timeout", "180"]

docker-compose.yml

@@ -24,12 +24,12 @@ services:
     restart: always
     depends_on:
       - elasticsearch
-    command: "/bin/bash -c 'sleep 15 && gunicorn rest_api.application:app -b 0.0.0.0 -k uvicorn.workers.UvicornWorker --workers 1 --timeout 180'"
+    command: "/bin/bash -c 'sleep 15 && gunicorn rest_api.application:app -b 0.0.0.0 -k uvicorn.workers.UvicornWorker --workers 1 --timeout 180 --preload'"
   elasticsearch:
     # This will start an empty elasticsearch instance (so you have to add your documents yourself)
     image: "elasticsearch:7.6.1"
     # If you want a demo image instead that is "ready-to-query" with some indexed Game of Thrones articles:
-    # image: "deepset/elasticsearch-game-of-thrones"
+    #image: "deepset/elasticsearch-game-of-thrones"
     ports:
       - 9200:9200
     environment:

haystack/document_store/elasticsearch.py

@@ -5,6 +5,7 @@ from string import Template
 from typing import List, Optional, Union, Dict, Any
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import bulk, scan
+from elasticsearch.exceptions import RequestError
 import numpy as np
 from scipy.special import expit
 
@@ -139,7 +140,16 @@ class ElasticsearchDocumentStore(BaseDocumentStore):
             }
             if self.embedding_field:
                 mapping["mappings"]["properties"][self.embedding_field] = {"type": "dense_vector", "dims": self.embedding_dim}
-        self.client.indices.create(index=index_name, body=mapping)
+
+        try:
+            self.client.indices.create(index=index_name, body=mapping)
+        except RequestError as e:
+            # With multiple workers we need to avoid race conditions, where:
+            # - there's no index in the beginning
+            # - both want to create one
+            # - one fails as the other one already created it
+            if not self.client.indices.exists(index=index_name):
+                raise e
 
     def _create_label_index(self, index_name):
         if self.client.indices.exists(index=index_name):
@@ -160,7 +170,15 @@ class ElasticsearchDocumentStore(BaseDocumentStore):
                 }
             }
         }
-        self.client.indices.create(index=index_name, body=mapping)
+        try:
+            self.client.indices.create(index=index_name, body=mapping)
+        except RequestError as e:
+            # With multiple workers we need to avoid race conditions, where:
+            # - there's no index in the beginning
+            # - both want to create one
+            # - one fails as the other one already created it
+            if not self.client.indices.exists(index=index_name):
+                raise e
 
     # TODO: Add flexibility to define other non-meta and meta fields expected by the Document class
     def _create_document_field_map(self) -> Dict:

requirements.txt

@@ -12,11 +12,11 @@ tox
 coverage
 langdetect # for PDF conversions
 # optional: sentence-transformers
-#temporarily (used for DPR downloads)
-wget
 python-multipart
 python-docx
 sqlalchemy_utils
 # for using FAISS with GPUs, install faiss-gpu
 faiss-cpu
 tika
+uvloop
+httptools

rest_api/config.py

@@ -7,7 +7,7 @@ PROJECT_NAME = os.getenv("PROJECT_NAME", "FastAPI")
 # Resources / Computation
 USE_GPU = os.getenv("USE_GPU", "True").lower() == "true"
 GPU_NUMBER = int(os.getenv("GPU_NUMBER", 1))
-MAX_PROCESSES = int(os.getenv("MAX_PROCESSES", 4))
+MAX_PROCESSES = int(os.getenv("MAX_PROCESSES", 0))
 BATCHSIZE = int(os.getenv("BATCHSIZE", 50))
 CONCURRENT_REQUEST_PER_WORKER = int(os.getenv("CONCURRENT_REQUEST_PER_WORKER", 4))
 
@@ -39,7 +39,7 @@ MAX_SEQ_LEN = int(os.getenv("MAX_SEQ_LEN", 256))
 
 # Retriever
 RETRIEVER_TYPE = os.getenv("RETRIEVER_TYPE", "ElasticsearchRetriever") # alternatives: 'EmbeddingRetriever', 'ElasticsearchRetriever', 'ElasticsearchFilterOnlyRetriever', None
-DEFAULT_TOP_K_RETRIEVER = int(os.getenv("DEFAULT_TOP_K_RETRIEVER", 10))
+DEFAULT_TOP_K_RETRIEVER = int(os.getenv("DEFAULT_TOP_K_RETRIEVER", 5))
 EXCLUDE_META_DATA_FIELDS = os.getenv("EXCLUDE_META_DATA_FIELDS", f"['question_emb','embedding']")
 if EXCLUDE_META_DATA_FIELDS:
     EXCLUDE_META_DATA_FIELDS = ast.literal_eval(EXCLUDE_META_DATA_FIELDS)

run_docker_gpu.sh

@@ -6,6 +6,9 @@
 # To use GPU with Docker, ensure nvidia-docker(https://github.com/NVIDIA/nvidia-docker) is installed.
 
 docker run -d -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.6.1
+# alternative: for a demo you can also use this elasticsearch image with already indexed GoT articles
+#docker run -d -p 9200:9200 -e "discovery.type=single-node" deepset/elasticsearch-game-of-thrones
+
 # wait for Elasticsearch server to start
 sleep 30
-docker run --net=host --gpus all -e READER_MODEL_PATH=deepset/roberta-base-squad2 -d deepset/haystack-gpu:0.2.0
+docker run --net=host --gpus all -e READER_MODEL_PATH=deepset/roberta-base-squad2 -d deepset/haystack-gpu:latest