diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml deleted file mode 100644 index e099fb643..000000000 --- a/.github/workflows/compliance.yml +++ /dev/null @@ -1,191 +0,0 @@ -name: License Compliance Checks - -on: - pull_request: - paths: - - "**/pyproject.toml" - schedule: - - cron: "0 0 * * *" # every day at midnight - -env: - GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - -jobs: - check-license-compliance-cpu: - if: ${{ !github.event.pull_request.head.repo.fork }} - name: Check CPU dependencies - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install Haystack (CPU) - # FIXME audio temporarily not checked as we plan to remove it in the near future - run: | - pip install --upgrade pip - pip install .[docstores,crawler,preprocessing,ocr,ray,onnx,beir] - pip install rest_api/ - - - name: Create file with full dependency list - run: | - pip freeze > requirements-full.txt - - - name: Send license report to Fossa - # This will collect all necessary information (mostly used dependencies) and send it to the Fossa API - uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred - with: - api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }} - - - name: Check license compliance - # This will poll the Fossa API until they have processed the information which we've sent in the previous step - # and fail if Fossa found an issue with the licences of our dependencies. - uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred - with: - api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }} - run-tests: true - - - uses: slackapi/slack-github-action@v1.23.0 - if: failure() && github.ref == 'refs/heads/main' - with: - payload: | - { - "blocks": [ - { - "type": "context", - "elements": [ - { - "type": "image", - "image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4", - "alt_text": "Actor" - }, - { - "type": "mrkdwn", - "text": "**" - } - ] - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Job ${{ github.job }} in workflow " - } - }, - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`" - } - ] - }, - { - "type": "context", - "elements": [ - { - "type": "image", - "image_url": "https://github.githubassets.com/favicons/favicon.png", - "alt_text": "Github logo" - }, - { - "type": "mrkdwn", - "text": " Run " - } - ] - } - ] - } - - check-license-compliance-gpu: - if: ${{ !github.event.pull_request.head.repo.fork }} - name: Check GPU dependencies - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install Haystack - # FIXME audio temporarily not checked as we plan to remove it in the near future - run: | - pip install --upgrade pip - pip install .[docstores-gpu,crawler,preprocessing,ocr,ray,onnx-gpu] - - - name: Create file with full dependency list - run: | - pip freeze > requirements-full.txt - - - name: Send license report to Fossa - # This will collect all necessary information (mostly used dependencies) and send it to the Fossa API - uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred - with: - api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }} - - - name: Check license compliance - # This will poll the Fossa API until they have processed the information which we've sent in the previous step - # and fail if Fossa found an issue with the licences of our dependencies. - uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred - with: - api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }} - run-tests: true - - - uses: slackapi/slack-github-action@v1.23.0 - if: failure() && github.ref == 'refs/heads/main' - with: - payload: | - { - "blocks": [ - { - "type": "context", - "elements": [ - { - "type": "image", - "image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4", - "alt_text": "Actor" - }, - { - "type": "mrkdwn", - "text": "**" - } - ] - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Job ${{ github.job }} in workflow " - } - }, - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`" - } - ] - }, - { - "type": "context", - "elements": [ - { - "type": "image", - "image_url": "https://github.githubassets.com/favicons/favicon.png", - "alt_text": "Github logo" - }, - { - "type": "mrkdwn", - "text": " Run " - } - ] - } - ] - } diff --git a/.github/workflows/license_compliance.yml b/.github/workflows/license_compliance.yml new file mode 100644 index 000000000..646dd224d --- /dev/null +++ b/.github/workflows/license_compliance.yml @@ -0,0 +1,295 @@ +name: License Compliance + +on: + pull_request: + paths: + - "**/pyproject.toml" + # Since we test PRs, there is no need to run the workflow at each + # merge on `main`. Let's use a cron job instead. + schedule: + - cron: "0 0 * * *" # every day at midnight + + +jobs: + license_check_vanilla: + name: Core dependencies + env: + REQUIREMENTS_FILE: requirements_vanilla.txt + runs-on: ubuntu-latest + steps: + - name: Checkout the code + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Get explicit and transitive dependencies + run: | + pip install . + pip freeze > ${{ env.REQUIREMENTS_FILE }} + + - name: Check Licenses + id: license_check_report + uses: pilosus/action-pip-license-checker@v2 + with: + github-token: ${{ secrets.GH_ACCESS_TOKEN }} + requirements: ${{ env.REQUIREMENTS_FILE }} + fail: 'Copyleft,Other,Error' + # Exclusions in the vanilla distribution must be explicitly motivated + # + # - certifi is pulled in by requests + # - num2words is pulled in by quantulum3 + # - tqdm is MLP but there are no better alternatives + # - nvidia libraries are brought in by torch on Linux, + # FIXME: to be removed once we stop depending on torch with the vanilla install + exclude: '(?i)^(certifi|num2words|tqdm|nvidia-).*' + + - name: Print report + if: ${{ always() }} + run: echo "${{ steps.license_check_report.outputs.report }}" + + - uses: slackapi/slack-github-action@v1.23.0 + if: failure() && github.ref == 'refs/heads/main' + with: + payload: | + { + "blocks": [ + { + "type": "context", + "elements": [ + { + "type": "image", + "image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4", + "alt_text": "Actor" + }, + { + "type": "mrkdwn", + "text": "**" + } + ] + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Job ${{ github.job }} in workflow " + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`" + } + ] + }, + { + "type": "context", + "elements": [ + { + "type": "image", + "image_url": "https://github.githubassets.com/favicons/favicon.png", + "alt_text": "Github logo" + }, + { + "type": "mrkdwn", + "text": " Run " + } + ] + } + ] + } + + + license_check_all: + name: All dependencies + env: + REQUIREMENTS_FILE: requirements_all.txt + runs-on: ubuntu-latest + steps: + - name: Checkout the code + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Get explicit and transitive dependencies + run: | + pip install -U pip + pip install .[all] + pip freeze > ${{ env.REQUIREMENTS_FILE }} + + - name: Check Licenses + id: license_check_report + uses: pilosus/action-pip-license-checker@v2 + with: + github-token: ${{ secrets.GH_ACCESS_TOKEN }} + requirements: ${{ env.REQUIREMENTS_FILE }} + fail: 'Copyleft,Other,Error' + # We allow incompatible licenses when they come from optional dependencies. + # + # Special cases: + # - pyzmq is flagged because dual-licensed, but we assume using BSD + # - tqdm is MLP but there are no better alternatives + exclude: '(?i)^(astroid|certifi|chardet|num2words|nvidia-|pathspec|pinecone-client|psycopg2|pylint|PyMuPDF|pyzmq|tqdm).*' + + - name: Print report + if: ${{ always() }} + run: echo "${{ steps.license_check_report.outputs.report }}" + + - uses: slackapi/slack-github-action@v1.23.0 + if: failure() && github.ref == 'refs/heads/main' + with: + payload: | + { + "blocks": [ + { + "type": "context", + "elements": [ + { + "type": "image", + "image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4", + "alt_text": "Actor" + }, + { + "type": "mrkdwn", + "text": "**" + } + ] + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Job ${{ github.job }} in workflow " + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`" + } + ] + }, + { + "type": "context", + "elements": [ + { + "type": "image", + "image_url": "https://github.githubassets.com/favicons/favicon.png", + "alt_text": "Github logo" + }, + { + "type": "mrkdwn", + "text": " Run " + } + ] + } + ] + } + + + license_check_all_GPU: + name: All dependencies - GPU + env: + REQUIREMENTS_FILE: requirements_all_gpu.txt + runs-on: ubuntu-latest + steps: + - name: Checkout the code + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Get explicit and transitive dependencies + run: | + pip install -U pip + pip install .[all-gpu] + pip freeze > ${{ env.REQUIREMENTS_FILE }} + + - name: Check Licenses + id: license_check_report + uses: pilosus/action-pip-license-checker@v2 + with: + github-token: ${{ secrets.GH_ACCESS_TOKEN }} + requirements: ${{ env.REQUIREMENTS_FILE }} + fail: 'Copyleft,Other,Error' + # We allow incompatible licenses when they come from optional dependencies. + # + # Special cases: + # - pyzmq is flagged because dual-licensed, but we assume using BSD + # - tqdm is MLP but there are no better alternatives + exclude: '(?i)^(astroid|certifi|chardet|num2words|nvidia-|pathspec|pinecone-client|psycopg2|pylint|PyMuPDF|pyzmq|tqdm).*' + + # We keep the license inventory on FOSSA + - name: Send license report to Fossa + uses: fossas/fossa-action@v1.3.1 + continue-on-error: true # not critical + with: + api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }} + + - name: Print report + if: ${{ always() }} + run: echo "${{ steps.license_check_report.outputs.report }}" + + - uses: slackapi/slack-github-action@v1.23.0 + if: failure() && github.ref == 'refs/heads/main' + with: + payload: | + { + "blocks": [ + { + "type": "context", + "elements": [ + { + "type": "image", + "image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4", + "alt_text": "Actor" + }, + { + "type": "mrkdwn", + "text": "**" + } + ] + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Job ${{ github.job }} in workflow " + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`" + } + ] + }, + { + "type": "context", + "elements": [ + { + "type": "image", + "image_url": "https://github.githubassets.com/favicons/favicon.png", + "alt_text": "Github logo" + }, + { + "type": "mrkdwn", + "text": " Run " + } + ] + } + ] + }