ci: add license compliance check (#3221)

* ci: add license compliance check

* ci: run check always for testing purposes

* revamp workflows

* temporary remove path directive

* triggering ci

* check rest api and ui too

* avoid cache to make sure env is clean

* add shield on readme

* ci: trigger CI to get latest scan

Co-authored-by: ZanSara <sarazanzo94@gmail.com>
Co-authored-by: Sara Zan <sara.zanzottera@deepset.ai>

.github/workflows/compliance.yml

@@ -0,0 +1,98 @@
+name: License Compliance Checks
+
+on:
+  pull_request:
+    paths:
+      - '**/pyproject.toml'
+  schedule:
+    - cron: '0 0 * * *'  # every day at midnight
+
+env:
+  GH_ACCESS_TOKEN:  ${{ secrets.GH_ACCESS_TOKEN }}
+  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  SLACK_ALERT_CHANNEL: "#haystack"
+
+jobs:
+  check-license-compliance-cpu:
+    name: Check CPU dependencies
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.7
+
+      - name: Install Haystack (CPU)
+        # FIXME audio temporarily not checked as we plan to remove it in the near future
+        run: |
+          pip install --upgrade pip
+          pip install .[docstores,crawler,preprocessing,ocr,ray,onnx,beir]
+          pip install rest_api/
+          pip install ui/
+
+      - name: Create file with full dependency list
+        run: |
+          pip freeze > requirements-full.txt
+
+      - name: Send license report to Fossa
+        # This will collect all necessary information (mostly used dependencies) and send it to the Fossa API
+        uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
+        with:
+          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
+
+      - name: Check license compliance
+        # This will poll the Fossa API until they have processed the information which we've sent in the previous step
+        # and fail if Fossa found an issue with the licences of our dependencies.
+        uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
+        with:
+          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
+          run-tests: true
+
+      - name: Send Slack notification if license check failed
+        uses: act10ns/slack@87c73aef9f8838eb6feae81589a6b1487a4a9e08
+        if: failure() && github.ref == 'refs/heads/master'
+        with:
+          status: ${{ job.status }}
+          channel: ${{ env.SLACK_ALERT_CHANNEL }}
+
+  check-license-compliance-gpu:
+    name: Check GPU dependencies
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.7
+
+      - name: Install Haystack
+        # FIXME audio temporarily not checked as we plan to remove it in the near future
+        run: |
+          pip install --upgrade pip
+          pip install .[docstores-gpu,crawler,preprocessing,ocr,ray,onnx-gpu]
+
+      - name: Create file with full dependency list
+        run: |
+          pip freeze > requirements-full.txt
+
+      - name: Send license report to Fossa
+        # This will collect all necessary information (mostly used dependencies) and send it to the Fossa API
+        uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
+        with:
+          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
+
+      - name: Check license compliance
+        # This will poll the Fossa API until they have processed the information which we've sent in the previous step
+        # and fail if Fossa found an issue with the licences of our dependencies.
+        uses: fossas/fossa-action@f61a4c0c263690f2ddb54b9822a719c25a7b608f # Use a specific version if locking is preferred
+        with:
+          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}
+          run-tests: true
+
+      - name: Send Slack notification if license check failed
+        uses: act10ns/slack@87c73aef9f8838eb6feae81589a6b1487a4a9e08
+        if: failure() && github.ref == 'refs/heads/master'
+        with:
+          status: ${{ job.status }}
+          channel: ${{ env.SLACK_ALERT_CHANNEL }}

README.md

@@ -12,6 +12,9 @@
     <a href="https://docs.haystack.deepset.ai">
         <img alt="Documentation" src="https://img.shields.io/website?label=documentation&up_message=online&url=https%3A%2F%2Fdocs.haystack.deepset.ai">
     </a>
+    <a href="https://app.fossa.com/projects/custom%2B24445%2Fgithub.com%2Fdeepset-ai%2Fhaystack?ref=badge_shield">
+        <img alt="FOSSA Status" src="https://app.fossa.com/api/projects/custom%2B24445%2Fgithub.com%2Fdeepset-ai%2Fhaystack.svg?type=shield"/>
+    </a>
     <a href="https://github.com/deepset-ai/haystack/releases">
         <img alt="Release" src="https://img.shields.io/github/release/deepset-ai/haystack">
     </a>

pyproject.toml

@@ -310,7 +310,6 @@ max-args=7
 [tool.pylint.'SIMILARITIES']
 min-similarity-lines=6
 
-
 [tool.pytest.ini_options]
 minversion = "6.0"
 addopts = "--strict-markers"

rest_api/pyproject.toml

@@ -66,9 +66,7 @@ python = ["37", "38", "39", "310"]
 [tool.coverage.run]
 branch = true
 parallel = true
-omit = [
+omit = ["rest_api/__about__.py"]
-  "rest_api/__about__.py",
-]
 
 [tool.coverage.report]
 exclude_lines = [

ui/pyproject.toml

@@ -57,9 +57,7 @@ python = ["37", "38", "39", "310"]
 [tool.coverage.run]
 branch = true
 parallel = true
-omit = [
+omit = ["ui/__about__.py"]
-  "ui/__about__.py",
-]
 
 [tool.coverage.report]
 exclude_lines = [