mirror of
https://github.com/deepset-ai/haystack.git
synced 2026-01-08 21:28:00 +00:00
d38c07e0ee
* Add knowledge graph module * Fix type hint * Add graph retriver module * Change type annotations, change return format * Add graph retriever that executes questions as sparql queries * Linking only those entities that are in the knowledge graph * Added logging and using relations extracted from Knowledge graph for linking * Preventing entity linking from linking the same token to multiple entities * Pruning triples that have no variables for select and count queries * Support knowledge graphs with Pipelines * Add text2sparql * Entity linking and relation linking consider more special cases now based on evaluation on labelled data * Separating example code from KGQA implementation * Add eval on combined extarctive and kg questions * Remove references to hp-test * Add fields sparql_query and long_answer_list to metadata * Removing modular Question2SPARQL approach * Removing additional classes used for modular kgqa approach * preparing lcquad data * change graph db * Translating namespaces in knowledge graph queries * Creating graphdb index and loading triples from .ttl file * Fetching graph config files, triples and model from S3 * Fix incompatibility issues with BaseGraphRetriever and BaseComponent * Removing unused utility functions * Adding doc strings and tutorial header * Adding sparqlwrapper dependency * Moving tutorial header * Sorting tutorials by number within name of notebook * Add latest docstring and tutorial changes * Creating test cases for knowledge graph * Changing knowledge graph example to harry potter * Add latest docstring and tutorial changes * Adapting the tutorial notebook to harry potter example * Add GraphDB fixture for tests * Add latest docstring and tutorial changes * Added GraphDB docker launch to CI * Use correct GraphDB fixture * Check if GraphDB instance is already running * Renaming question/query and incorporating other feedback from Timo and Tanay * Removed type annotation * Add latest docstring and tutorial changes Co-authored-by: oryx1729 <oryx1729@protonmail.com> Co-authored-by: Timo Moeller <timo.moeller@deepset.ai> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
90 lines
3.0 KiB
YAML
90 lines
3.0 KiB
YAML
name: Build
|
|
|
|
on:
|
|
push:
|
|
branches: [ master ]
|
|
pull_request:
|
|
branches: [ master ]
|
|
|
|
jobs:
|
|
type-check:
|
|
runs-on: ubuntu-20.04
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- uses: actions/setup-python@v2
|
|
with:
|
|
python-version: 3.7
|
|
- name: Test with mypy
|
|
run: |
|
|
pip install mypy
|
|
mypy haystack --ignore-missing-imports
|
|
|
|
build-cache:
|
|
needs: type-check
|
|
runs-on: ubuntu-20.04
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- uses: actions/setup-python@v2
|
|
with:
|
|
python-version: 3.7
|
|
- run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
|
- name: Cache
|
|
id: cache-python-env
|
|
uses: actions/cache@v2
|
|
with:
|
|
path: ${{ env.pythonLocation }}
|
|
key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }}
|
|
- name: Install dependencies
|
|
if: steps.cache-python-env.outputs.cache-hit != 'true'
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install --upgrade --upgrade-strategy eager -r requirements-dev.txt -e .
|
|
pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
|
|
|
|
prepare-build:
|
|
needs: build-cache
|
|
runs-on: ubuntu-20.04
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- id: set-matrix
|
|
run: |
|
|
echo "::set-output name=matrix::$(cd test && ls -d test_*.py | jq -R . | jq -cs .)"
|
|
outputs:
|
|
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
|
build:
|
|
needs: prepare-build
|
|
runs-on: ubuntu-20.04
|
|
strategy:
|
|
matrix:
|
|
test-path: ${{fromJson(needs.prepare-build.outputs.matrix)}}
|
|
fail-fast: false
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: Set up Python 3.7
|
|
uses: actions/setup-python@v2
|
|
with:
|
|
python-version: 3.7
|
|
- run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
|
- name: Cache
|
|
uses: actions/cache@v2
|
|
with:
|
|
path: ${{ env.pythonLocation }}
|
|
key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }}
|
|
- name: Run Elasticsearch
|
|
run: docker run -d -p 9200:9200 -e "discovery.type=single-node" -e "ES_JAVA_OPTS=-Xms128m -Xmx128m" elasticsearch:7.9.2
|
|
|
|
- name: Run Milvus
|
|
run: docker run -d -p 19530:19530 -p 19121:19121 milvusdb/milvus:1.0.0-cpu-d030521-1ea92e
|
|
|
|
- name: Run GraphDB
|
|
run: docker run -d -p 7200:7200 --name haystack_test_graphdb docker-registry.ontotext.com/graphdb-free:9.4.1-adoptopenjdk11
|
|
|
|
- name: Run Apache Tika
|
|
run: docker run -d -p 9998:9998 -e "TIKA_CHILD_JAVA_OPTS=-JXms128m" -e "TIKA_CHILD_JAVA_OPTS=-JXmx128m" apache/tika:1.24.1
|
|
|
|
- name: Install pdftotext
|
|
run: wget --no-check-certificate https://dl.xpdfreader.com/xpdf-tools-linux-4.03.tar.gz && tar -xvf xpdf-tools-linux-4.03.tar.gz && sudo cp xpdf-tools-linux-4.03/bin64/pdftotext /usr/local/bin
|
|
|
|
- name: Run tests
|
|
run: cd test && pytest ${{ matrix.test-path }}
|