mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-12-31 09:10:15 +00:00
0acafc403a
* initial test cml * Update cml.yaml * WIP test workflow * switch to general ubuntu ami * switch to general ubuntu ami * disable gpu for tests * rm gpu infos * rm gpu infos * update token env * switch github token * add postgres * test db connection * fix typo * remove tty * add sleep for db * debug runner * debug removal postgres * debug: reset to working commit * debug: change github token * switch to new bot token * debug token * add back postgres * adjust network runner docker * add elastic * fix typo * adjust working dir * fix benchmark execution * enable s3 downloads * add query benchmark. fix path * add saving of markdown files * cat md files. add faiss+dpr. increase n_queries * switch to GPU instance * switch availability zone * switch to public aws DL ami * increase volume size * rm faiss. fix error logging * save markdown files * add reader benchmarks * add download of squad data * correct reader metric normalization * fix newlines between reports * fix max_docs for reader eval data. remove max_docs from ci run config * fix mypy. switch workflow trigger * try trigger for label * try trigger for label * change trigger syntax * debug machine shutdown with test workflow * add es and postgres to test workflow * Revert "add es and postgres to test workflow" This reverts commit 6f038d3d7f12eea924b54529e61b192858eaa9d5. * Revert "debug machine shutdown with test workflow" This reverts commit db70eabae8850b88e1d61fd79b04d4f49d54990a. * fix typo in action. set benchmark config back to original
91 lines
3.7 KiB
YAML
91 lines
3.7 KiB
YAML
name: benchmarks
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
pull_request:
|
|
types: [labeled]
|
|
jobs:
|
|
deploy-cloud-runner:
|
|
if: ${{ (github.event.action == 'labeled' && github.event.label.name == 'benchmark') || github.event.action == 'workflow_dispatch' }}
|
|
runs-on: [ubuntu-latest]
|
|
container: docker://dvcorg/cml
|
|
steps:
|
|
- name: deploy
|
|
env:
|
|
repo_token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_ACCESS_KEY }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_SECRET_ACCESS_KEY }}
|
|
VPC: ${{ secrets.AWS_CI_VPC }}
|
|
run: |
|
|
echo "Deploying..."
|
|
RUNNER_LABELS="cml,aws"
|
|
RUNNER_REPO="https://github.com/${GITHUB_REPOSITORY}"
|
|
MACHINE="cml$(date +%s)"
|
|
docker-machine create \
|
|
--driver amazonec2 \
|
|
--amazonec2-instance-type p3.2xlarge \
|
|
--amazonec2-vpc-id $VPC \
|
|
--amazonec2-region us-east-1 \
|
|
--amazonec2-zone c \
|
|
--amazonec2-ssh-user ubuntu \
|
|
--amazonec2-ami ami-06a25ee8966373068 \
|
|
--amazonec2-root-size 150 \
|
|
$MACHINE
|
|
eval "$(docker-machine env --shell sh $MACHINE)"
|
|
|
|
(
|
|
docker-machine ssh $MACHINE "sudo mkdir -p \
|
|
/docker_machine && \
|
|
sudo chmod 777 /docker_machine" && \
|
|
docker-machine scp -r -q ~/.docker/machine/ \
|
|
$MACHINE:/docker_machine && \
|
|
docker run --name elasticsearch -d \
|
|
-p 9200:9200 \
|
|
-e "discovery.type=single-node" \
|
|
elasticsearch:7.9.2 && \
|
|
docker run --name postgres -d \
|
|
-p 5432:5432 \
|
|
--net host \
|
|
-e POSTGRES_PASSWORD=password \
|
|
-v /docker_machine/machine:/root/.docker/machine \
|
|
-e DOCKER_MACHINE=$MACHINE \
|
|
postgres && \
|
|
sleep 4 && \
|
|
docker exec -i postgres psql -U postgres -c "CREATE DATABASE haystack;" && \
|
|
docker run --name runner -d \
|
|
--gpus all \
|
|
-v /docker_machine/machine:/root/.docker/machine \
|
|
--net host \
|
|
-e DOCKER_MACHINE=$MACHINE \
|
|
-e repo_token=$repo_token \
|
|
-e RUNNER_LABELS=$RUNNER_LABELS \
|
|
-e RUNNER_REPO=$RUNNER_REPO \
|
|
-e RUNNER_IDLE_TIMEOUT=120 \
|
|
dvcorg/cml-py3:latest && \
|
|
sleep 20 && echo "Deployed $MACHINE"
|
|
) || (echo "Shut down machine" && docker-machine rm -y -f $MACHINE && exit 1)
|
|
run-benchmark:
|
|
if: ${{ (github.event.action == 'labeled' && github.event.label.name == 'benchmark') || github.event.action == 'workflow_dispatch' }}
|
|
needs: deploy-cloud-runner
|
|
runs-on: [self-hosted,cml]
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: cml_run
|
|
env:
|
|
repo_token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_ACCESS_KEY }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_SECRET_ACCESS_KEY }}
|
|
run: |
|
|
apt-get update -y
|
|
apt-get install python3-dev -y
|
|
pip install -r requirements.txt
|
|
pip install .
|
|
cd test/benchmarks && python run.py --retriever_index --retriever_query --reader --ci --save_markdown
|
|
echo -en "## Benchmarks: Retriever Indexing\n" >> report.md
|
|
cat retriever_index_results.md >> report.md
|
|
echo -en "\n\n## Benchmarks: Retriever Querying\n" >> report.md
|
|
cat retriever_query_results.md >> report.md
|
|
echo -en "\n\n## Benchmarks: Reader\n" >> report.md
|
|
cat reader_results.md >> report.md
|
|
cml-send-comment report.md
|