haystack/.github/workflows/license_compliance.yml

name: License Compliance

on:
  pull_request:
    paths:
      - "**/pyproject.toml"
  # Since we test PRs, there is no need to run the workflow at each
  # merge on `main`. Let's use a cron job instead.
  schedule:
    - cron: "0 0 * * *" # every day at midnight

env:
  CORE_DATADOG_API_KEY: ${{ secrets.CORE_DATADOG_API_KEY }}

jobs:
  license_check_direct:
    name: Direct dependencies only
    env:
      REQUIREMENTS_FILE: requirements_direct.txt
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the code
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Get direct dependencies, all extras
        run: |
          pip install toml
          python .github/utils/pyproject_to_requirements.py pyproject.toml --extra all > ${{ env.REQUIREMENTS_FILE }}

      - name: Check Licenses
        id: license_check_report
        uses: pilosus/action-pip-license-checker@v2
        with:
          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
          requirements: ${{ env.REQUIREMENTS_FILE }}
          fail: "Copyleft,Other,Error"
          # Exclusions in the vanilla distribution must be explicitly motivated
          #
          # - tqdm is MLP but there are no better alternatives
          # - PyMuPDF is optional
          # - pinecone-client is optional
          # - psycopg2 is optional
          exclude: "(?i)^(PyMuPDF|tqdm|pinecone-client|psycopg2).*"

      # We keep the license inventory on FOSSA
      - name: Send license report to Fossa
        uses: fossas/fossa-action@v1.3.1
        continue-on-error: true # not critical
        with:
          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}

      - name: Print report
        if: ${{ always() }}
        run: echo "${{ steps.license_check_report.outputs.report }}"

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure())
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        # This step would fail when running in PRs opened from forks since
        # secrets are not accessible.
        # To prevent showing bogus failures in those PRs we skip the step.
        # The workflow will fail in any case if the actual check fails in the previous steps.
        if: (success() || failure()) && env.CORE_DATADOG_API_KEY != ''
        uses: masci/datadog@v1
        with:
          api-key: ${{ env.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.job }} in ${{ github.workflow }} workflow"
              text: "License compliance check: direct dependencies only."
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  license_check_vanilla:
    name: Core dependencies, no extras
    env:
      REQUIREMENTS_FILE: requirements_vanilla.txt
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the code
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Get explicit and transitive dependencies
        run: |
          pip install .
          pip freeze > ${{ env.REQUIREMENTS_FILE }}

      - name: Check Licenses
        id: license_check_report
        uses: pilosus/action-pip-license-checker@v2
        with:
          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
          requirements: ${{ env.REQUIREMENTS_FILE }}
          fail: "Copyleft,Other,Error"
          # Exclusions in the vanilla distribution must be explicitly motivated
          #
          # - certifi is pulled in by requests
          # - num2words is pulled in by quantulum3
          # - tqdm is MLP but there are no better alternatives
          # - nvidia libraries are brought in by torch on Linux,
          #   FIXME: to be removed once we stop depending on torch with the vanilla install
          exclude: "(?i)^(certifi|num2words|tqdm|nvidia-).*"

      - name: Print report
        if: ${{ always() }}
        run: echo "${{ steps.license_check_report.outputs.report }}"

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure())
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        # This step would fail when running in PRs opened from forks since
        # secrets are not accessible.
        # To prevent showing bogus failures in those PRs we skip the step.
        # The workflow will fail in any case if the actual check fails in the previous steps.
        if: (success() || failure()) && env.CORE_DATADOG_API_KEY != ''
        uses: masci/datadog@v1
        with:
          api-key: ${{ env.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.job }} in ${{ github.workflow }} workflow"
              text: "License compliance check: core dependencies, no extras."
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  license_check_all:
    name: All extras
    env:
      REQUIREMENTS_FILE: requirements_all.txt
    runs-on: ubuntu-latest-4-cores
    steps:
      - name: Checkout the code
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Get explicit and transitive dependencies
        run: |
          pip install -U pip
          pip install .[all]
          pip freeze > ${{ env.REQUIREMENTS_FILE }}

      - name: Check Licenses
        id: license_check_report
        uses: pilosus/action-pip-license-checker@v2
        with:
          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
          requirements: ${{ env.REQUIREMENTS_FILE }}
          fail: "Copyleft,Other,Error"
          # We allow incompatible licenses when they come from optional dependencies.
          #
          # Special cases:
          # - pyzmq is flagged because dual-licensed, but we assume using BSD
          # - tqdm is MLP but there are no better alternatives
          exclude: "(?i)^(astroid|certifi|chardet|num2words|nvidia-|pathspec|pinecone-client|psycopg2|pylint|PyMuPDF|pyzmq|tqdm).*"

      - name: Print report
        if: ${{ always() }}
        run: echo "${{ steps.license_check_report.outputs.report }}"

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure())
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        # This step would fail when running in PRs opened from forks since
        # secrets are not accessible.
        # To prevent showing bogus failures in those PRs we skip the step.
        # The workflow will fail in any case if the actual check fails in the previous steps.
        if: (success() || failure()) && env.CORE_DATADOG_API_KEY != ''
        uses: masci/datadog@v1
        with:
          api-key: ${{ env.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.job }} in ${{ github.workflow }} workflow"
              text: "License compliance check: all available extras."
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  license_check_all_GPU:
    name: All extras, GPU version
    env:
      REQUIREMENTS_FILE: requirements_all_gpu.txt
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the code
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Get explicit and transitive dependencies
        run: |
          pip install -U pip
          pip install .[all-gpu]
          pip freeze > ${{ env.REQUIREMENTS_FILE }}

      - name: Check Licenses
        id: license_check_report
        uses: pilosus/action-pip-license-checker@v2
        with:
          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
          requirements: ${{ env.REQUIREMENTS_FILE }}
          fail: "Copyleft,Other,Error"
          # We allow incompatible licenses when they come from optional dependencies.
          #
          # Special cases:
          # - pyzmq is flagged because dual-licensed, but we assume using BSD
          # - tqdm is MLP but there are no better alternatives
          exclude: "(?i)^(astroid|certifi|chardet|num2words|nvidia-|pathspec|pinecone-client|psycopg2|pylint|PyMuPDF|pyzmq|tqdm).*"

      - name: Print report
        if: ${{ always() }}
        run: echo "${{ steps.license_check_report.outputs.report }}"

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure())
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        # This step would fail when running in PRs opened from forks since
        # secrets are not accessible.
        # To prevent showing bogus failures in those PRs we skip the step.
        # The workflow will fail in any case if the actual check fails in the previous steps.
        if: (success() || failure()) && env.CORE_DATADOG_API_KEY != ''
        uses: masci/datadog@v1
        with:
          api-key: ${{ env.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.job }} in ${{ github.workflow }} workflow"
              text: "License compliance check: all available extras, GPU version."
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"