[issue-1976] - Ingestion SonarCloud (#2085)

* Update path

* Prepare sonar properties

* Prepare coverage recipes

* Add coverage

* Simplify pytest

* Organise integration tests

* Update path

* Use setup instead of reqs

* Update recipes

* Fix PR event to target

* Update event_name

* Prepare sonar

* Run tests & sonar

* Use sonarcloud host

* Fix compose

* Use ingestion token

.github/workflows/maven-build.yml

@@ -17,7 +17,7 @@ on:
     paths:
       - "catalog-rest-service/**"
       - "openmetadata-ui/**"
-  pull_request:
+  pull_request_target:
     branches: [main]
     paths:
       - "catalog-rest-service/**"
@@ -52,7 +52,7 @@ jobs:
       - name: Build PR with Maven
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        if: ${{ github.event_name == 'pull_request' }}
+        if: ${{ github.event_name == 'pull_request_target' }}
         run: |
           git fetch --no-tags https://github.com/open-metadata/OpenMetadata.git +refs/heads/main:refs/remotes/origin/main
           mvn -Dsonar.login=${{ secrets.SONAR_TOKEN }} \

.github/workflows/py-tests.yml

@@ -16,12 +16,12 @@ on:
     branches: [main]
     paths:
       - ingestion/**
-  pull_request:
+  pull_request_target:
     branches: [main]
     paths:
       - ingestion/**
 jobs:
-  run_integration_tests:
+  run_tests:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -30,22 +30,48 @@ jobs:
       with:
         java-version: '11'
         distribution: 'adopt'
+
     - name: Start Server and Ingest Sample Data
       run: ./docker/run_local_docker.sh
       timeout-minutes: 20
+
     - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
         python-version: 3.9
+
     - name: Install Ubuntu dependencies
       run: |
         sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv
+
     - name: Install open-metadata dependencies
       run: |
         python3 -m venv env
         source env/bin/activate
-        make install_test install_dev generate install
-    - name: Run Python Integration Tests
+        make install_test install_dev generate
+
+    - name: Run Python Tests & record coverage
       run: |
         source env/bin/activate
-        make run_ometa_integration_tests
+        make coverage
+
+    - name: Run PR Sonar
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      if: ${{ github.event_name == 'pull_request_target' }}
+      run: |
+        git fetch --no-tags https://github.com/open-metadata/OpenMetadata.git +refs/heads/main:refs/remotes/origin/main
+
+        # Add PR analysis information
+        echo "\nsonar.pullrequest.key=${{ github.event.pull_request.number }}" >> ingestion/sonar-project.properties
+        echo "sonar.pullrequest.branch=${{ github.head_ref }}" >> ingestion/sonar-project.properties
+        echo "sonar.pullrequest.base=origin/main" >> ingestion/sonar-project.properties
+        echo "sonar.pullrequest.provider=github" >> ingestion/sonar-project.properties
+
+        make token=${{ secrets.INGESTION_SONAR_SECRET }} sonar_ingestion
+
+    - name: Run Sonar
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      if: ${{ github.event_name == 'push' }}
+      run: make token=${{ secrets.INGESTION_SONAR_SECRET }} sonar_ingestion

.gitignore

@@ -84,4 +84,4 @@ ingestion/.python-version
 
 # MLFlow
 mlruns/
-/ingestion/tests/integration/mlflow/tests/db/
+/ingestion/tests/integration/source/mlflow/tests/db/

Makefile

@@ -11,10 +11,10 @@ install:
 	python3 -m pip install ingestion/
 
 install_test:
-	python3 -m pip install -r ingestion/requirements-test.txt
+	python3 -m pip install "ingestion[test]/"
 
 install_dev:
-	python3 -m pip install -r ingestion/requirements-dev.txt
+	python3 -m pip install "ingestion[dev]/"
 
 precommit_install:
 	@echo "Installing pre-commit hooks"
@@ -39,8 +39,25 @@ generate:
 	datamodel-codegen  --input catalog-rest-service/src/main/resources/json  --input-file-type jsonschema --output ingestion/src/metadata/generated
 
 run_ometa_integration_tests:
-	cd ingestion; \
-	pytest -c setup.cfg --override-ini=testpaths="tests/integration/ometa tests/unit/stage_test.py"
+	coverage run -m pytest -c ingestion/setup.cfg --doctest-modules --junitxml=ingestion/junit/test-results-integration.xml --override-ini=testpaths="ingestion/tests/integration/ometa ingestion/tests/integration/stage"
+
+unit_ingestion:
+	coverage run -m pytest -c ingestion/setup.cfg -s --doctest-modules --junitxml=ingestion/junit/test-results-unit.xml --override-ini=testpaths="ingestion/tests/unit"
+
+coverage:
+	coverage erase
+	make unit_ingestion
+	make run_ometa_integration_tests
+	coverage xml -i -o ingestion/coverage.xml
+
+sonar_ingestion:
+	docker run \
+		--rm \
+		-e SONAR_HOST_URL="https://sonarcloud.io" \
+		-e SONAR_LOGIN=$(token) \
+		-v ${PWD}:/usr/src \
+		sonarsource/sonar-scanner-cli \
+		-Dproject.settings=ingestion/sonar-project.properties
 
 publish:
 	make install_dev generate

ingestion/requirements-dev.txt

@@ -1,6 +0,0 @@
-boto3==1.20.14
-botocore==1.23.14
-datamodel-code-generator==0.11.14
-docker
-google-cloud-storage==1.43.0
-twine

ingestion/requirements-test.txt

@@ -1,7 +0,0 @@
-black
-isort
-pre-commit
-pylint
-pytest
-pytest-cov
-faker

ingestion/setup.cfg

@@ -33,7 +33,6 @@ indent='    '
 sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
 
 [tool:pytest]
-addopts = --cov src --cov-report term --cov-config setup.cfg --strict-markers
 markers =
     slow: marks tests as slow (deselect with '-m "not slow"')
 testpaths =

ingestion/setup.py

@@ -108,6 +108,24 @@ plugins: Dict[str, Set[str]] = {
     "okta": {"okta~=2.3.0"},
     "mlflow": {"mlflow-skinny~=1.22.0"},
 }
+dev = {
+    "boto3==1.20.14",
+    "botocore==1.23.14",
+    "datamodel-code-generator==0.11.14",
+    "docker",
+    "google-cloud-storage==1.43.0",
+    "twine",
+}
+test = {
+    "black",
+    "isort",
+    "pre-commit",
+    "pylint",
+    "pytest",
+    "pytest-cov",
+    "faker",
+    "coverage",
+}
 
 build_options = {"includes": ["_cffi_backend"]}
 setup(
@@ -138,6 +156,8 @@ setup(
     install_requires=list(base_requirements),
     extras_require={
         "base": list(base_requirements),
+        "dev": list(dev),
+        "test": list(test),
         **{plugin: list(dependencies) for (plugin, dependencies) in plugins.items()},
         "all": list(
             base_requirements.union(

ingestion/sonar-project.properties

@@ -0,0 +1,11 @@
+sonar.projectKey=openmetadata-ingestion
+sonar.projectName=openmetadata-ingestion
+sonar.organization=open-metadata
+sonar.language=py
+
+sonar.sources=ingestion/src
+sonar.tests=ingestion/tests
+sonar.exclusions=ingestion/src/metadata_server/static/**,ingestion/src/metadata_server/templates/**
+sonar.python.xunit.reportPath=ingestion/junit/test-results-*.xml
+sonar.python.coverage.reportPaths=ingestion/coverage.xml
+sonar.python.version=3.8,3.9

ingestion/tests/integration/source/hive/tests/docker-compose.yml

@@ -19,7 +19,7 @@ services:
     environment:
       - CLUSTER_NAME=test
     env_file:
-      - ./hadoop-hive.env
+      - hadoop-hive.env
     ports:
       - "50070:50070"
   datanode:
@@ -27,7 +27,7 @@ services:
     volumes:
       - datanode:/hadoop/dfs/data
     env_file:
-      - ./hadoop-hive.env
+      - hadoop-hive.env
     environment:
       SERVICE_PRECONDITION: "namenode:50070"
     ports:
@@ -35,7 +35,7 @@ services:
   hive-server:
     image: bde2020/hive:2.3.2-postgresql-metastore
     env_file:
-      - ./hadoop-hive.env
+      - hadoop-hive.env
     command: "sh /setup/setup.sh"
     volumes:
       - ./setup:/setup
@@ -47,7 +47,7 @@ services:
   hive-metastore:
     image: bde2020/hive:2.3.2-postgresql-metastore
     env_file:
-      - ./hadoop-hive.env
+      - hadoop-hive.env
     command: /opt/hive/bin/hive --service metastore
     environment:
       SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432"

ingestion/tests/integration/stage/stage_test.py

@@ -8,7 +8,7 @@ config = """
   "source": {
     "type": "sample-data",
     "config": {
-      "sample_data_folder": "examples/sample_data/"
+      "sample_data_folder": "ingestion/examples/sample_data"
     }
   },
   "stage": {

ingestion/tests/unit/test_config.py

@@ -24,7 +24,7 @@ class TestConfig(TestCase):
     Check config reading
     """
 
-    basedir = os.path.join("resources", "config")
+    basedir = os.path.join(os.path.dirname(__file__), "resources", "config")
 
     def test_basic(self):
         """