ci: add new license checker (#4779)

* try

* add exclusions

* fix vanilla distribution

* use different requirements files

* fix comments and file name

* try with a recent version of pip

* use cpu version of torch

* try

* again

* exclude nvidia libraries

* revert old change

* send report to FOSSA

* add gpu section

* display job names

* remove FOSSA check

* send complete report to FOSSA

* removed FIXME

.github/workflows/compliance.yml

@@ -1,191 +0,0 @@
-name: License Compliance Checks
-
-on:
-  pull_request:
-    paths:
-      - "**/pyproject.toml"
-  schedule:
-    - cron: "0 0 * * *" # every day at midnight
-
-env:
-  GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
-  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-  SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
-
-jobs:
-  check-license-compliance-cpu:
-    if: ${{ !github.event.pull_request.head.repo.fork }}
-    name: Check CPU dependencies
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-
-      - name: Install Haystack (CPU)
-        # FIXME audio temporarily not checked as we plan to remove it in the near future
-        run: |
-          pip install --upgrade pip
-          pip install .[docstores,crawler,preprocessing,ocr,ray,onnx,beir]
-          pip install rest_api/
-
-      - name: Create file with full dependency list
-        run: |
-          pip freeze > requirements-full.txt
-
-      - name: Send license report to Fossa
-        # This will collect all necessary information (mostly used dependencies) and send it to the Fossa API
-        uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
-        with:
-          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
-
-      - name: Check license compliance
-        # This will poll the Fossa API until they have processed the information which we've sent in the previous step
-        # and fail if Fossa found an issue with the licences of our dependencies.
-        uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
-        with:
-          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
-          run-tests: true
-
-      - uses: slackapi/slack-github-action@v1.23.0
-        if: failure() && github.ref == 'refs/heads/main'
-        with:
-          payload: |
-            {
-              "blocks": [
-                {
-                  "type": "context",
-                  "elements": [
-                    {
-                      "type": "image",
-                      "image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4",
-                      "alt_text": "Actor"
-                    },
-                    {
-                      "type": "mrkdwn",
-                      "text": "*<https://github.com/${{ github.actor }}|${{ github.actor }}>*"
-                    }
-                  ]
-                },
-                {
-                  "type": "section",
-                  "text": {
-                    "type": "mrkdwn",
-                    "text": "Job ${{ github.job }} in workflow <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/workflow/|${{ github.workflow }}>"
-                  }
-                },
-                {
-                  "type": "context",
-                  "elements": [
-                    {
-                      "type": "mrkdwn",
-                      "text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`"
-                    }
-                  ]
-                },
-                {
-                  "type": "context",
-                  "elements": [
-                    {
-                      "type": "image",
-                      "image_url": "https://github.githubassets.com/favicons/favicon.png",
-                      "alt_text": "Github logo"
-                    },
-                    {
-                      "type": "mrkdwn",
-                      "text": "<https://github.com/${{ github.repository }}/|${{ github.repository }}> Run <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/|#${{ github.run_number }} | Attempt #${{ github.run_attempt }}>"
-                    }
-                  ]
-                }
-              ]
-            }
-
-  check-license-compliance-gpu:
-    if: ${{ !github.event.pull_request.head.repo.fork }}
-    name: Check GPU dependencies
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-
-      - name: Install Haystack
-        # FIXME audio temporarily not checked as we plan to remove it in the near future
-        run: |
-          pip install --upgrade pip
-          pip install .[docstores-gpu,crawler,preprocessing,ocr,ray,onnx-gpu]
-
-      - name: Create file with full dependency list
-        run: |
-          pip freeze > requirements-full.txt
-
-      - name: Send license report to Fossa
-        # This will collect all necessary information (mostly used dependencies) and send it to the Fossa API
-        uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
-        with:
-          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
-
-      - name: Check license compliance
-        # This will poll the Fossa API until they have processed the information which we've sent in the previous step
-        # and fail if Fossa found an issue with the licences of our dependencies.
-        uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
-        with:
-          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
-          run-tests: true
-
-      - uses: slackapi/slack-github-action@v1.23.0
-        if: failure() && github.ref == 'refs/heads/main'
-        with:
-          payload: |
-            {
-              "blocks": [
-                {
-                  "type": "context",
-                  "elements": [
-                    {
-                      "type": "image",
-                      "image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4",
-                      "alt_text": "Actor"
-                    },
-                    {
-                      "type": "mrkdwn",
-                      "text": "*<https://github.com/${{ github.actor }}|${{ github.actor }}>*"
-                    }
-                  ]
-                },
-                {
-                  "type": "section",
-                  "text": {
-                    "type": "mrkdwn",
-                    "text": "Job ${{ github.job }} in workflow <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/workflow/|${{ github.workflow }}>"
-                  }
-                },
-                {
-                  "type": "context",
-                  "elements": [
-                    {
-                      "type": "mrkdwn",
-                      "text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`"
-                    }
-                  ]
-                },
-                {
-                  "type": "context",
-                  "elements": [
-                    {
-                      "type": "image",
-                      "image_url": "https://github.githubassets.com/favicons/favicon.png",
-                      "alt_text": "Github logo"
-                    },
-                    {
-                      "type": "mrkdwn",
-                      "text": "<https://github.com/${{ github.repository }}/|${{ github.repository }}> Run <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/|#${{ github.run_number }} | Attempt #${{ github.run_attempt }}>"
-                    }
-                  ]
-                }
-              ]
-            }

.github/workflows/license_compliance.yml

@@ -0,0 +1,295 @@
+name: License Compliance
+
+on:
+  pull_request:
+    paths:
+      - "**/pyproject.toml"
+  # Since we test PRs, there is no need to run the workflow at each
+  # merge on `main`. Let's use a cron job instead.
+  schedule:
+    - cron: "0 0 * * *" # every day at midnight
+
+
+jobs:
+  license_check_vanilla:
+    name: Core dependencies
+    env:
+      REQUIREMENTS_FILE: requirements_vanilla.txt
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout the code
+      uses: actions/checkout@v3
+
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: Get explicit and transitive dependencies
+      run: |
+        pip install .
+        pip freeze > ${{ env.REQUIREMENTS_FILE }}
+
+    - name: Check Licenses
+      id: license_check_report
+      uses: pilosus/action-pip-license-checker@v2
+      with:
+        github-token: ${{ secrets.GH_ACCESS_TOKEN }}
+        requirements: ${{ env.REQUIREMENTS_FILE }}
+        fail: 'Copyleft,Other,Error'
+        # Exclusions in the vanilla distribution must be explicitly motivated
+        #
+        # - certifi is pulled in by requests
+        # - num2words is pulled in by quantulum3
+        # - tqdm is MLP but there are no better alternatives
+        # - nvidia libraries are brought in by torch on Linux,
+        #   FIXME: to be removed once we stop depending on torch with the vanilla install
+        exclude: '(?i)^(certifi|num2words|tqdm|nvidia-).*'
+
+    - name: Print report
+      if: ${{ always() }}
+      run: echo "${{ steps.license_check_report.outputs.report }}"
+
+    - uses: slackapi/slack-github-action@v1.23.0
+      if: failure() && github.ref == 'refs/heads/main'
+      with:
+        payload: |
+          {
+            "blocks": [
+              {
+                "type": "context",
+                "elements": [
+                  {
+                    "type": "image",
+                    "image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4",
+                    "alt_text": "Actor"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*<https://github.com/${{ github.actor }}|${{ github.actor }}>*"
+                  }
+                ]
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "Job ${{ github.job }} in workflow <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/workflow/|${{ github.workflow }}>"
+                }
+              },
+              {
+                "type": "context",
+                "elements": [
+                  {
+                    "type": "mrkdwn",
+                    "text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`"
+                  }
+                ]
+              },
+              {
+                "type": "context",
+                "elements": [
+                  {
+                    "type": "image",
+                    "image_url": "https://github.githubassets.com/favicons/favicon.png",
+                    "alt_text": "Github logo"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "<https://github.com/${{ github.repository }}/|${{ github.repository }}> Run <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/|#${{ github.run_number }} | Attempt #${{ github.run_attempt }}>"
+                  }
+                ]
+              }
+            ]
+          }
+
+
+  license_check_all:
+    name: All dependencies
+    env:
+      REQUIREMENTS_FILE: requirements_all.txt
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout the code
+      uses: actions/checkout@v3
+
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: Get explicit and transitive dependencies
+      run: |
+        pip install -U pip
+        pip install .[all]
+        pip freeze > ${{ env.REQUIREMENTS_FILE }}
+
+    - name: Check Licenses
+      id: license_check_report
+      uses: pilosus/action-pip-license-checker@v2
+      with:
+        github-token: ${{ secrets.GH_ACCESS_TOKEN }}
+        requirements: ${{ env.REQUIREMENTS_FILE }}
+        fail: 'Copyleft,Other,Error'
+        # We allow incompatible licenses when they come from optional dependencies.
+        #
+        # Special cases:
+        # - pyzmq is flagged because dual-licensed, but we assume using BSD
+        # - tqdm is MLP but there are no better alternatives
+        exclude: '(?i)^(astroid|certifi|chardet|num2words|nvidia-|pathspec|pinecone-client|psycopg2|pylint|PyMuPDF|pyzmq|tqdm).*'
+
+    - name: Print report
+      if: ${{ always() }}
+      run: echo "${{ steps.license_check_report.outputs.report }}"
+
+    - uses: slackapi/slack-github-action@v1.23.0
+      if: failure() && github.ref == 'refs/heads/main'
+      with:
+        payload: |
+          {
+            "blocks": [
+              {
+                "type": "context",
+                "elements": [
+                  {
+                    "type": "image",
+                    "image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4",
+                    "alt_text": "Actor"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*<https://github.com/${{ github.actor }}|${{ github.actor }}>*"
+                  }
+                ]
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "Job ${{ github.job }} in workflow <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/workflow/|${{ github.workflow }}>"
+                }
+              },
+              {
+                "type": "context",
+                "elements": [
+                  {
+                    "type": "mrkdwn",
+                    "text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`"
+                  }
+                ]
+              },
+              {
+                "type": "context",
+                "elements": [
+                  {
+                    "type": "image",
+                    "image_url": "https://github.githubassets.com/favicons/favicon.png",
+                    "alt_text": "Github logo"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "<https://github.com/${{ github.repository }}/|${{ github.repository }}> Run <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/|#${{ github.run_number }} | Attempt #${{ github.run_attempt }}>"
+                  }
+                ]
+              }
+            ]
+          }
+
+
+  license_check_all_GPU:
+    name: All dependencies - GPU
+    env:
+      REQUIREMENTS_FILE: requirements_all_gpu.txt
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout the code
+      uses: actions/checkout@v3
+
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: Get explicit and transitive dependencies
+      run: |
+        pip install -U pip
+        pip install .[all-gpu]
+        pip freeze > ${{ env.REQUIREMENTS_FILE }}
+
+    - name: Check Licenses
+      id: license_check_report
+      uses: pilosus/action-pip-license-checker@v2
+      with:
+        github-token: ${{ secrets.GH_ACCESS_TOKEN }}
+        requirements: ${{ env.REQUIREMENTS_FILE }}
+        fail: 'Copyleft,Other,Error'
+        # We allow incompatible licenses when they come from optional dependencies.
+        #
+        # Special cases:
+        # - pyzmq is flagged because dual-licensed, but we assume using BSD
+        # - tqdm is MLP but there are no better alternatives
+        exclude: '(?i)^(astroid|certifi|chardet|num2words|nvidia-|pathspec|pinecone-client|psycopg2|pylint|PyMuPDF|pyzmq|tqdm).*'
+
+    # We keep the license inventory on FOSSA
+    - name: Send license report to Fossa
+      uses: fossas/fossa-action@v1.3.1
+      continue-on-error: true  # not critical
+      with:
+        api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
+
+    - name: Print report
+      if: ${{ always() }}
+      run: echo "${{ steps.license_check_report.outputs.report }}"
+
+    - uses: slackapi/slack-github-action@v1.23.0
+      if: failure() && github.ref == 'refs/heads/main'
+      with:
+        payload: |
+          {
+            "blocks": [
+              {
+                "type": "context",
+                "elements": [
+                  {
+                    "type": "image",
+                    "image_url": "https://avatars.githubusercontent.com/u/${{ github.actor_id }}?v=4",
+                    "alt_text": "Actor"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*<https://github.com/${{ github.actor }}|${{ github.actor }}>*"
+                  }
+                ]
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "Job ${{ github.job }} in workflow <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/workflow/|${{ github.workflow }}>"
+                }
+              },
+              {
+                "type": "context",
+                "elements": [
+                  {
+                    "type": "mrkdwn",
+                    "text": "Triggered by ${{ github.event_name }} for ${{ github.ref_type }} `${{ github.ref_name }}`"
+                  }
+                ]
+              },
+              {
+                "type": "context",
+                "elements": [
+                  {
+                    "type": "image",
+                    "image_url": "https://github.githubassets.com/favicons/favicon.png",
+                    "alt_text": "Github logo"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "<https://github.com/${{ github.repository }}/|${{ github.repository }}> Run <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/|#${{ github.run_number }} | Attempt #${{ github.run_attempt }}>"
+                  }
+                ]
+              }
+            ]
+          }