Merging PyTests with Github Actions CI with matrix strategy into a single file (#7628)

* Delete py-tests-3_10.yml

Merging PyTests with Github Actions CI with matrix strategy into a single file #7355

* Delete py-tests-3_8.yml

Merging PyTests with Github Actions CI with matrix strategy into a single file #7355

* Delete py-tests-3_9.yml

Merging PyTests with Github Actions CI with matrix strategy into a single file #7355

* Merging PyTest files into a single file

Merging PyTests with Github Actions CI with matrix strategy into a single file #7355

.github/workflows/py-tests-3_10.yml

@@ -1,72 +0,0 @@
-
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-name: py-tests-3.10
-on:
-  push:
-    branches: [main]
-    paths:
-      - ingestion/**
-      - openmetadata-service/**
-  pull_request_target:
-    branches: [main]
-    paths:
-      - ingestion/**
-      - openmetadata-service/**
-
-jobs:
-  py-run-tests:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-
-    - name: Set up JDK 11
-      uses: actions/setup-java@v2
-      with:
-        java-version: '11'
-        distribution: 'adopt'
-
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v2
-      with:
-        python-version: "3.10"
-
-    - name: Install Ubuntu dependencies
-      run: |
-        sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
-        unixodbc-dev libevent-dev python3-dev
-
-    - name: Generate models
-      run: |
-        python3 -m venv env
-        source env/bin/activate
-        sudo make install_antlr_cli
-        make install_dev generate
-
-    - name: Install open-metadata dependencies
-      run: |
-        source env/bin/activate
-        make install_all install_test
-
-    - name: Start Server and Ingest Sample Data
-      env:
-        INGESTION_DEPENDENCY: "mysql,elasticsearch"
-      run: ./docker/run_local_docker.sh -m no-ui
-      timeout-minutes: 30
-
-    - name: Run Python Tests & record coverage
-      run: |
-        source env/bin/activate
-        make coverage

.github/workflows/py-tests-3_8.yml

@@ -1,72 +0,0 @@
-
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-name: py-tests-3.8
-on:
-  push:
-    branches: [main]
-    paths:
-      - ingestion/**
-      - openmetadata-service/**
-  pull_request_target:
-    branches: [main]
-    paths:
-      - ingestion/**
-      - openmetadata-service/**
-
-jobs:
-  py-run-tests:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-
-    - name: Set up JDK 11
-      uses: actions/setup-java@v2
-      with:
-        java-version: '11'
-        distribution: 'adopt'
-
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-
-    - name: Install Ubuntu dependencies
-      run: |
-        sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
-        librdkafka-dev unixodbc-dev libevent-dev
-
-    - name: Generate models
-      run: |
-        python3 -m venv env
-        source env/bin/activate
-        sudo make install_antlr_cli
-        make install_dev generate
-
-    - name: Install open-metadata dependencies
-      run: |
-        source env/bin/activate
-        make install_all install_test
-
-    - name: Start Server and Ingest Sample Data
-      env:
-        INGESTION_DEPENDENCY: "mysql,elasticsearch"
-      run: ./docker/run_local_docker.sh -m no-ui
-      timeout-minutes: 30
-
-    - name: Run Python Tests
-      run: |
-        source env/bin/activate
-        make run_python_tests

.github/workflows/py-tests-3_9.yml

@@ -1,105 +0,0 @@
-
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-name: py-tests
-on:
-  push:
-    branches: [main]
-    paths-ignore:
-      - 'openmetadata-docs/**'
-  pull_request_target:
-    types: [labeled, opened, synchronize, reopened]
-    paths-ignore:
-      - 'openmetadata-docs/**'
-
-concurrency: 
-  group: py-tests-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-jobs:
-  py-run-tests:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-        fetch-depth: 0
-
-    - name: Set up JDK 11
-      uses: actions/setup-java@v2
-      with:
-        java-version: '11'
-        distribution: 'adopt'
-
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.9
-
-    - name: Install Ubuntu dependencies
-      run: |
-        # stop relying on apt cache of GitHub runners
-        sudo apt-get update
-        sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
-        librdkafka-dev unixodbc-dev libevent-dev
-
-    - name: Generate models
-      run: |
-        python3 -m venv env
-        source env/bin/activate
-        sudo make install_antlr_cli
-        make install_dev generate
-
-    - name: Install open-metadata dependencies
-      run: |
-        source env/bin/activate
-        make install_all install_test
-
-    - name: Start Server and Ingest Sample Data
-      env:
-        INGESTION_DEPENDENCY: "mysql,elasticsearch"
-      run: ./docker/run_local_docker.sh -m no-ui
-      timeout-minutes: 30
-
-    - name: Run Python Tests & Record Coverage
-      run: |
-        source env/bin/activate
-        make coverage
-        rm pom.xml
-        # fix coverage xml report for github
-        sed -i 's/src\/metadata/\/github\/workspace\/ingestion\/src\/metadata/g' ingestion/ci-coverage.xml
-
-    # we have to pass these args values since we are working with the 'pull_request_target' trigger
-    - name: Push Results in PR to Sonar
-      uses: sonarsource/sonarcloud-github-action@master
-      if: ${{ github.event_name == 'pull_request_target' }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        SONAR_TOKEN: ${{ secrets.INGESTION_SONAR_SECRET }}
-      with:
-        projectBaseDir: ingestion/
-        args: >
-          -Dproject.settings=ingestion/sonar-project.properties
-          -Dsonar.pullrequest.key=${{ github.event.pull_request.number }}
-          -Dsonar.pullrequest.branch=${{ github.head_ref }}
-          -Dsonar.pullrequest.github.repository=OpenMetadata
-          -Dsonar.scm.revision=${{ github.event.pull_request.head.sha }}
-          -Dsonar.pullrequest.provider=github
-
-    - name: Push Results to Sonar
-      uses: sonarsource/sonarcloud-github-action@master
-      if: ${{ github.event_name == 'push' }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        SONAR_TOKEN: ${{ secrets.INGESTION_SONAR_SECRET }}
-      with:
-        projectBaseDir: ingestion/

.github/workflows/py-tests.yml

@@ -1,4 +1,3 @@
-
 #  Copyright 2021 Collate
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -10,7 +9,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-name: py-tests-3.7
+name: py-tests
 on:
   push:
     branches: [main]
@@ -26,6 +25,9 @@ on:
 jobs:
   py-run-tests:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        py-version: ['3.7', '3.8', '3.9', '3.10']
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -38,10 +40,10 @@ jobs:
         java-version: '11'
         distribution: 'adopt'
 
-    - name: Set up Python 3.7
+    - name: Set up Python ${{ matrix.py-version }}
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version:  ${{ matrix.py-version }}
 
     - name: Install Ubuntu dependencies
       run: |