mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-09-26 16:46:58 +00:00
ci: add new license checker (#4779)
* try * add exclusions * fix vanilla distribution * use different requirements files * fix comments and file name * try with a recent version of pip * use cpu version of torch * try * again * exclude nvidia libraries * revert old change * send report to FOSSA * add gpu section * display job names * remove FOSSA check * send complete report to FOSSA * removed FIXME
This commit is contained in:
Massimiliano Pippi
GitHub
parent
eff420cce0
commit
c619aa29ec
191
.github/workflows/compliance.yml
vendored
191
.github/workflows/compliance.yml
vendored
@ -1,191 +0,0 @@
|
|||||||
name: License Compliance Checks
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
paths:
|
|
||||||
- "**/pyproject.toml"
|
|
||||||
schedule:
|
|
||||||
- cron: "0 0 * * *" # every day at midnight
|
|
||||||
|
|
||||||
env:
|
|
||||||
GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
|
|
||||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
|
||||||
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
check-license-compliance-cpu:
|
|
||||||
if: ${{ !github.event.pull_request.head.repo.fork }}
|
|
||||||
name: Check CPU dependencies
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
|
|
||||||
- uses: actions/setup-python@v4
|
|
||||||
with:
|
|
||||||
python-version: 3.8
|
|
||||||
|
|
||||||
- name: Install Haystack (CPU)
|
|
||||||
# FIXME audio temporarily not checked as we plan to remove it in the near future
|
|
||||||
run: |
|
|
||||||
pip install --upgrade pip
|
|
||||||
pip install .[docstores,crawler,preprocessing,ocr,ray,onnx,beir]
|
|
||||||
pip install rest_api/
|
|
||||||
|
|
||||||
- name: Create file with full dependency list
|
|
||||||
run: |
|
|
||||||
pip freeze > requirements-full.txt
|
|
||||||
|
|
||||||
- name: Send license report to Fossa
|
|
||||||
# This will collect all necessary information (mostly used dependencies) and send it to the Fossa API
|
|
||||||
uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
|
|
||||||
with:
|
|
||||||
api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
|
|
||||||
|
|
||||||
- name: Check license compliance
|
|
||||||
# This will poll the Fossa API until they have processed the information which we've sent in the previous step
|
|
||||||
# and fail if Fossa found an issue with the licences of our dependencies.
|
|
||||||
uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
|
|
||||||
with:
|
|
||||||
api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
|
|
||||||
run-tests: true
|
|
||||||
|
|
||||||
- uses: slackapi/slack-github-action@v1.23.0
|
|
||||||
if: failure() && github.ref == 'refs/heads/main'
|
|
||||||
with:
|
|
||||||
payload: |
|
|
||||||
{
|
|
||||||
"blocks": [
|
|
||||||
{
|
|
||||||
"type": "context",
|
|
||||||
"elements": [
|
|
||||||
{
|
|
||||||
"type": "image",
|
|
||||||
"image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4",
|
|
||||||
"alt_text": "Actor"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "mrkdwn",
|
|
||||||
"text": "*<https://github.com/${{ github.actor }}|${{ github.actor }}>*"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "section",
|
|
||||||
"text": {
|
|
||||||
"type": "mrkdwn",
|
|
||||||
"text": "Job ${{ github.job }} in workflow <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/workflow/|${{ github.workflow }}>"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "context",
|
|
||||||
"elements": [
|
|
||||||
{
|
|
||||||
"type": "mrkdwn",
|
|
||||||
"text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "context",
|
|
||||||
"elements": [
|
|
||||||
{
|
|
||||||
"type": "image",
|
|
||||||
"image_url": "https://github.githubassets.com/favicons/favicon.png",
|
|
||||||
"alt_text": "Github logo"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "mrkdwn",
|
|
||||||
"text": "<https://github.com/${{ github.repository }}/|${{ github.repository }}> Run <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/|#${{ github.run_number }} | Attempt #${{ github.run_attempt }}>"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
check-license-compliance-gpu:
|
|
||||||
if: ${{ !github.event.pull_request.head.repo.fork }}
|
|
||||||
name: Check GPU dependencies
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v3
|
|
||||||
|
|
||||||
- uses: actions/setup-python@v4
|
|
||||||
with:
|
|
||||||
python-version: 3.8
|
|
||||||
|
|
||||||
- name: Install Haystack
|
|
||||||
# FIXME audio temporarily not checked as we plan to remove it in the near future
|
|
||||||
run: |
|
|
||||||
pip install --upgrade pip
|
|
||||||
pip install .[docstores-gpu,crawler,preprocessing,ocr,ray,onnx-gpu]
|
|
||||||
|
|
||||||
- name: Create file with full dependency list
|
|
||||||
run: |
|
|
||||||
pip freeze > requirements-full.txt
|
|
||||||
|
|
||||||
- name: Send license report to Fossa
|
|
||||||
# This will collect all necessary information (mostly used dependencies) and send it to the Fossa API
|
|
||||||
uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
|
|
||||||
with:
|
|
||||||
api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
|
|
||||||
|
|
||||||
- name: Check license compliance
|
|
||||||
# This will poll the Fossa API until they have processed the information which we've sent in the previous step
|
|
||||||
# and fail if Fossa found an issue with the licences of our dependencies.
|
|
||||||
uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
|
|
||||||
with:
|
|
||||||
api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
|
|
||||||
run-tests: true
|
|
||||||
|
|
||||||
- uses: slackapi/slack-github-action@v1.23.0
|
|
||||||
if: failure() && github.ref == 'refs/heads/main'
|
|
||||||
with:
|
|
||||||
payload: |
|
|
||||||
{
|
|
||||||
"blocks": [
|
|
||||||
{
|
|
||||||
"type": "context",
|
|
||||||
"elements": [
|
|
||||||
{
|
|
||||||
"type": "image",
|
|
||||||
"image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4",
|
|
||||||
"alt_text": "Actor"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "mrkdwn",
|
|
||||||
"text": "*<https://github.com/${{ github.actor }}|${{ github.actor }}>*"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "section",
|
|
||||||
"text": {
|
|
||||||
"type": "mrkdwn",
|
|
||||||
"text": "Job ${{ github.job }} in workflow <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/workflow/|${{ github.workflow }}>"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "context",
|
|
||||||
"elements": [
|
|
||||||
{
|
|
||||||
"type": "mrkdwn",
|
|
||||||
"text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "context",
|
|
||||||
"elements": [
|
|
||||||
{
|
|
||||||
"type": "image",
|
|
||||||
"image_url": "https://github.githubassets.com/favicons/favicon.png",
|
|
||||||
"alt_text": "Github logo"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "mrkdwn",
|
|
||||||
"text": "<https://github.com/${{ github.repository }}/|${{ github.repository }}> Run <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/|#${{ github.run_number }} | Attempt #${{ github.run_attempt }}>"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
295
.github/workflows/license_compliance.yml
vendored
Normal file
295
.github/workflows/license_compliance.yml
vendored
Normal file
@ -0,0 +1,295 @@
|
|||||||
|
name: License Compliance
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- "**/pyproject.toml"
|
||||||
|
# Since we test PRs, there is no need to run the workflow at each
|
||||||
|
# merge on `main`. Let's use a cron job instead.
|
||||||
|
schedule:
|
||||||
|
- cron: "0 0 * * *" # every day at midnight
|
||||||
|
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
license_check_vanilla:
|
||||||
|
name: Core dependencies
|
||||||
|
env:
|
||||||
|
REQUIREMENTS_FILE: requirements_vanilla.txt
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout the code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Setup Python
|
||||||
|
uses: actions/setup-python@v4
|
||||||
|
with:
|
||||||
|
python-version: '3.10'
|
||||||
|
|
||||||
|
- name: Get explicit and transitive dependencies
|
||||||
|
run: |
|
||||||
|
pip install .
|
||||||
|
pip freeze > ${{ env.REQUIREMENTS_FILE }}
|
||||||
|
|
||||||
|
- name: Check Licenses
|
||||||
|
id: license_check_report
|
||||||
|
uses: pilosus/action-pip-license-checker@v2
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GH_ACCESS_TOKEN }}
|
||||||
|
requirements: ${{ env.REQUIREMENTS_FILE }}
|
||||||
|
fail: 'Copyleft,Other,Error'
|
||||||
|
# Exclusions in the vanilla distribution must be explicitly motivated
|
||||||
|
#
|
||||||
|
# - certifi is pulled in by requests
|
||||||
|
# - num2words is pulled in by quantulum3
|
||||||
|
# - tqdm is MLP but there are no better alternatives
|
||||||
|
# - nvidia libraries are brought in by torch on Linux,
|
||||||
|
# FIXME: to be removed once we stop depending on torch with the vanilla install
|
||||||
|
exclude: '(?i)^(certifi|num2words|tqdm|nvidia-).*'
|
||||||
|
|
||||||
|
- name: Print report
|
||||||
|
if: ${{ always() }}
|
||||||
|
run: echo "${{ steps.license_check_report.outputs.report }}"
|
||||||
|
|
||||||
|
- uses: slackapi/slack-github-action@v1.23.0
|
||||||
|
if: failure() && github.ref == 'refs/heads/main'
|
||||||
|
with:
|
||||||
|
payload: |
|
||||||
|
{
|
||||||
|
"blocks": [
|
||||||
|
{
|
||||||
|
"type": "context",
|
||||||
|
"elements": [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4",
|
||||||
|
"alt_text": "Actor"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "*<https://github.com/${{ github.actor }}|${{ github.actor }}>*"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "section",
|
||||||
|
"text": {
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "Job ${{ github.job }} in workflow <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/workflow/|${{ github.workflow }}>"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "context",
|
||||||
|
"elements": [
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "context",
|
||||||
|
"elements": [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"image_url": "https://github.githubassets.com/favicons/favicon.png",
|
||||||
|
"alt_text": "Github logo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "<https://github.com/${{ github.repository }}/|${{ github.repository }}> Run <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/|#${{ github.run_number }} | Attempt #${{ github.run_attempt }}>"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
license_check_all:
|
||||||
|
name: All dependencies
|
||||||
|
env:
|
||||||
|
REQUIREMENTS_FILE: requirements_all.txt
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout the code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Setup Python
|
||||||
|
uses: actions/setup-python@v4
|
||||||
|
with:
|
||||||
|
python-version: '3.10'
|
||||||
|
|
||||||
|
- name: Get explicit and transitive dependencies
|
||||||
|
run: |
|
||||||
|
pip install -U pip
|
||||||
|
pip install .[all]
|
||||||
|
pip freeze > ${{ env.REQUIREMENTS_FILE }}
|
||||||
|
|
||||||
|
- name: Check Licenses
|
||||||
|
id: license_check_report
|
||||||
|
uses: pilosus/action-pip-license-checker@v2
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GH_ACCESS_TOKEN }}
|
||||||
|
requirements: ${{ env.REQUIREMENTS_FILE }}
|
||||||
|
fail: 'Copyleft,Other,Error'
|
||||||
|
# We allow incompatible licenses when they come from optional dependencies.
|
||||||
|
#
|
||||||
|
# Special cases:
|
||||||
|
# - pyzmq is flagged because dual-licensed, but we assume using BSD
|
||||||
|
# - tqdm is MLP but there are no better alternatives
|
||||||
|
exclude: '(?i)^(astroid|certifi|chardet|num2words|nvidia-|pathspec|pinecone-client|psycopg2|pylint|PyMuPDF|pyzmq|tqdm).*'
|
||||||
|
|
||||||
|
- name: Print report
|
||||||
|
if: ${{ always() }}
|
||||||
|
run: echo "${{ steps.license_check_report.outputs.report }}"
|
||||||
|
|
||||||
|
- uses: slackapi/slack-github-action@v1.23.0
|
||||||
|
if: failure() && github.ref == 'refs/heads/main'
|
||||||
|
with:
|
||||||
|
payload: |
|
||||||
|
{
|
||||||
|
"blocks": [
|
||||||
|
{
|
||||||
|
"type": "context",
|
||||||
|
"elements": [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4",
|
||||||
|
"alt_text": "Actor"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "*<https://github.com/${{ github.actor }}|${{ github.actor }}>*"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "section",
|
||||||
|
"text": {
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "Job ${{ github.job }} in workflow <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/workflow/|${{ github.workflow }}>"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "context",
|
||||||
|
"elements": [
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "context",
|
||||||
|
"elements": [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"image_url": "https://github.githubassets.com/favicons/favicon.png",
|
||||||
|
"alt_text": "Github logo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "<https://github.com/${{ github.repository }}/|${{ github.repository }}> Run <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/|#${{ github.run_number }} | Attempt #${{ github.run_attempt }}>"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
license_check_all_GPU:
|
||||||
|
name: All dependencies - GPU
|
||||||
|
env:
|
||||||
|
REQUIREMENTS_FILE: requirements_all_gpu.txt
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout the code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Setup Python
|
||||||
|
uses: actions/setup-python@v4
|
||||||
|
with:
|
||||||
|
python-version: '3.10'
|
||||||
|
|
||||||
|
- name: Get explicit and transitive dependencies
|
||||||
|
run: |
|
||||||
|
pip install -U pip
|
||||||
|
pip install .[all-gpu]
|
||||||
|
pip freeze > ${{ env.REQUIREMENTS_FILE }}
|
||||||
|
|
||||||
|
- name: Check Licenses
|
||||||
|
id: license_check_report
|
||||||
|
uses: pilosus/action-pip-license-checker@v2
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GH_ACCESS_TOKEN }}
|
||||||
|
requirements: ${{ env.REQUIREMENTS_FILE }}
|
||||||
|
fail: 'Copyleft,Other,Error'
|
||||||
|
# We allow incompatible licenses when they come from optional dependencies.
|
||||||
|
#
|
||||||
|
# Special cases:
|
||||||
|
# - pyzmq is flagged because dual-licensed, but we assume using BSD
|
||||||
|
# - tqdm is MLP but there are no better alternatives
|
||||||
|
exclude: '(?i)^(astroid|certifi|chardet|num2words|nvidia-|pathspec|pinecone-client|psycopg2|pylint|PyMuPDF|pyzmq|tqdm).*'
|
||||||
|
|
||||||
|
# We keep the license inventory on FOSSA
|
||||||
|
- name: Send license report to Fossa
|
||||||
|
uses: fossas/fossa-action@v1.3.1
|
||||||
|
continue-on-error: true # not critical
|
||||||
|
with:
|
||||||
|
api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
|
||||||
|
|
||||||
|
- name: Print report
|
||||||
|
if: ${{ always() }}
|
||||||
|
run: echo "${{ steps.license_check_report.outputs.report }}"
|
||||||
|
|
||||||
|
- uses: slackapi/slack-github-action@v1.23.0
|
||||||
|
if: failure() && github.ref == 'refs/heads/main'
|
||||||
|
with:
|
||||||
|
payload: |
|
||||||
|
{
|
||||||
|
"blocks": [
|
||||||
|
{
|
||||||
|
"type": "context",
|
||||||
|
"elements": [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4",
|
||||||
|
"alt_text": "Actor"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "*<https://github.com/${{ github.actor }}|${{ github.actor }}>*"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "section",
|
||||||
|
"text": {
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "Job ${{ github.job }} in workflow <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/workflow/|${{ github.workflow }}>"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "context",
|
||||||
|
"elements": [
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "context",
|
||||||
|
"elements": [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"image_url": "https://github.githubassets.com/favicons/favicon.png",
|
||||||
|
"alt_text": "Github logo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "mrkdwn",
|
||||||
|
"text": "<https://github.com/${{ github.repository }}/|${{ github.repository }}> Run <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/|#${{ github.run_number }} | Attempt #${{ github.run_attempt }}>"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user