mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-07-22 16:31:16 +00:00

* first draft / notes on new primitives * wip label / feedback refactor * rename doc.text -> doc.content. add doc.content_type * add datatype for content * remove faq_question_field from ES and weaviate. rename text_field -> content_field in docstores. update tutorials for content field * update converters for . Add warning for empty * renam label.question -> label.query. Allow sorting of Answers. * WIP primitives * update ui/reader for new Answer format * Improve Label. First refactoring of MultiLabel. Adjust eval code * fixed workflow conflict with introducing new one (#1472) * Add latest docstring and tutorial changes * make add_eval_data() work again * fix reader formats. WIP fix _extract_docs_and_labels_from_dict * fix test reader * Add latest docstring and tutorial changes * fix another test case for reader * fix mypy in farm reader.eval() * fix mypy in farm reader.eval() * WIP ORM refactor * Add latest docstring and tutorial changes * fix mypy weaviate * make label and multilabel dataclasses * bump mypy env in CI to python 3.8 * WIP refactor Label ORM * WIP refactor Label ORM * simplify tests for individual doc stores * WIP refactoring markers of tests * test alternative approach for tests with existing parametrization * WIP refactor ORMs * fix skip logic of already parametrized tests * fix weaviate behaviour in tests - not parametrizing it in our general test cases. * Add latest docstring and tutorial changes * fix some tests * remove sql from document_store_types * fix markers for generator and pipeline test * remove inmemory marker * remove unneeded elasticsearch markers * add dataclasses-json dependency. adjust ORM to just store JSON repr * ignore type as dataclasses_json seems to miss functionality here * update readme and contributing.md * update contributing * adjust example * fix duplicate doc handling for custom index * Add latest docstring and tutorial changes * fix some ORM issues. fix get_all_labels_aggregated. * update drop flags where get_all_labels_aggregated() was used before * Add latest docstring and tutorial changes * add to_json(). add + fix tests * fix no_answer handling in label / multilabel * fix duplicate docs in memory doc store. change primary key for sql doc table * fix mypy issues * fix mypy issues * haystack/retriever/base.py * fix test_write_document_meta[elastic] * fix test_elasticsearch_custom_fields * fix test_labels[elastic] * fix crawler * fix converter * fix docx converter * fix preprocessor * fix test_utils * fix tfidf retriever. fix selection of docstore in tests with multiple fixtures / parameterizations * Add latest docstring and tutorial changes * fix crawler test. fix ocrconverter attribute * fix test_elasticsearch_custom_query * fix generator pipeline * fix ocr converter * fix ragenerator * Add latest docstring and tutorial changes * fix test_load_and_save_yaml for elasticsearch * fixes for pipeline tests * fix faq pipeline * fix pipeline tests * Add latest docstring and tutorial changes * fix weaviate * Add latest docstring and tutorial changes * trigger CI * satisfy mypy * Add latest docstring and tutorial changes * satisfy mypy * Add latest docstring and tutorial changes * trigger CI * fix question generation test * fix ray. fix Q-generation * fix translator test * satisfy mypy * wip refactor feedback rest api * fix rest api feedback endpoint * fix doc classifier * remove relation of Labels -> Docs in SQL ORM * fix faiss/milvus tests * fix doc classifier test * fix eval test * fixing eval issues * Add latest docstring and tutorial changes * fix mypy * WIP replace dataclasses-json with manual serialization * Add latest docstring and tutorial changes * revert to dataclass-json serialization for now. remove debug prints. * update docstrings * fix extractor. fix Answer Span init * fix api test * keep meta data of answers in reader.run() * fix meta handling * adress review feedback * Add latest docstring and tutorial changes * make document=None for open domain labels * add import * fix print utils * fix rest api * adress review feedback * Add latest docstring and tutorial changes * fix mypy Co-authored-by: Markus Paff <markuspaff.mp@gmail.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
71 lines
2.2 KiB
Python
71 lines
2.2 KiB
Python
import logging
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from fastapi import APIRouter
|
|
|
|
from haystack import Pipeline
|
|
from rest_api.config import PIPELINE_YAML_PATH, QUERY_PIPELINE_NAME
|
|
from rest_api.config import LOG_LEVEL, CONCURRENT_REQUEST_PER_WORKER
|
|
from rest_api.schema import QueryRequest, QueryResponse
|
|
from rest_api.controller.utils import RequestLimiter
|
|
|
|
|
|
logging.getLogger("haystack").setLevel(LOG_LEVEL)
|
|
logger = logging.getLogger("haystack")
|
|
|
|
from pydantic import BaseConfig
|
|
|
|
BaseConfig.arbitrary_types_allowed = True
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
PIPELINE = Pipeline.load_from_yaml(Path(PIPELINE_YAML_PATH), pipeline_name=QUERY_PIPELINE_NAME)
|
|
# TODO make this generic for other pipelines with different naming
|
|
RETRIEVER = PIPELINE.get_node(name="Retriever")
|
|
DOCUMENT_STORE = RETRIEVER.document_store if RETRIEVER else None
|
|
logging.info(f"Loaded pipeline nodes: {PIPELINE.graph.nodes.keys()}")
|
|
|
|
concurrency_limiter = RequestLimiter(CONCURRENT_REQUEST_PER_WORKER)
|
|
|
|
|
|
@router.get("/initialized")
|
|
def initialized():
|
|
"""
|
|
This endpoint can be used during startup to understand if the
|
|
server is ready to take any requests, or is still loading.
|
|
|
|
The recommended approach is to call this endpoint with a short timeout,
|
|
like 500ms, and in case of no reply, consider the server busy.
|
|
"""
|
|
return True
|
|
|
|
|
|
@router.post("/query", response_model=QueryResponse)
|
|
def query(request: QueryRequest):
|
|
with concurrency_limiter.run():
|
|
result = _process_request(PIPELINE, request)
|
|
return result
|
|
|
|
|
|
def _process_request(pipeline, request) -> QueryResponse:
|
|
start_time = time.time()
|
|
|
|
params = request.params or {}
|
|
params["filters"] = params.get("filters") or {}
|
|
filters = {}
|
|
if "filters" in params: # put filter values into a list and remove filters with null value
|
|
for key, values in params["filters"].items():
|
|
if values is None:
|
|
continue
|
|
if not isinstance(values, list):
|
|
values = [values]
|
|
filters[key] = values
|
|
params["filters"] = filters
|
|
result = pipeline.run(query=request.query, params=params)
|
|
end_time = time.time()
|
|
logger.info({"request": request.dict(), "response": result, "time": f"{(end_time - start_time):.2f}"})
|
|
|
|
return result
|