diff --git a/.github/workflows/maven-build.yml b/.github/workflows/maven-build.yml index 87f18ea6bb1..ebac00a0b97 100644 --- a/.github/workflows/maven-build.yml +++ b/.github/workflows/maven-build.yml @@ -17,7 +17,7 @@ on: paths: - "catalog-rest-service/**" - "openmetadata-ui/**" - pull_request: + pull_request_target: branches: [main] paths: - "catalog-rest-service/**" @@ -52,7 +52,7 @@ jobs: - name: Build PR with Maven env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: ${{ github.event_name == 'pull_request' }} + if: ${{ github.event_name == 'pull_request_target' }} run: | git fetch --no-tags https://github.com/open-metadata/OpenMetadata.git +refs/heads/main:refs/remotes/origin/main mvn -Dsonar.login=${{ secrets.SONAR_TOKEN }} \ diff --git a/.github/workflows/py-integration-tests.yml b/.github/workflows/py-tests.yml similarity index 55% rename from .github/workflows/py-integration-tests.yml rename to .github/workflows/py-tests.yml index b61d091ac6c..4e2cff764b6 100644 --- a/.github/workflows/py-integration-tests.yml +++ b/.github/workflows/py-tests.yml @@ -16,12 +16,12 @@ on: branches: [main] paths: - ingestion/** - pull_request: + pull_request_target: branches: [main] paths: - ingestion/** jobs: - run_integration_tests: + run_tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -30,22 +30,48 @@ jobs: with: java-version: '11' distribution: 'adopt' + - name: Start Server and Ingest Sample Data run: ./docker/run_local_docker.sh timeout-minutes: 20 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: python-version: 3.9 + - name: Install Ubuntu dependencies run: | sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv + - name: Install open-metadata dependencies run: | python3 -m venv env source env/bin/activate - make install_test install_dev generate install - - name: Run Python Integration Tests + make install_test install_dev generate + + - name: Run Python Tests & record coverage run: | source env/bin/activate - make run_ometa_integration_tests + make coverage + + - name: Run PR Sonar + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: ${{ github.event_name == 'pull_request_target' }} + run: | + git fetch --no-tags https://github.com/open-metadata/OpenMetadata.git +refs/heads/main:refs/remotes/origin/main + + # Add PR analysis information + echo "\nsonar.pullrequest.key=${{ github.event.pull_request.number }}" >> ingestion/sonar-project.properties + echo "sonar.pullrequest.branch=${{ github.head_ref }}" >> ingestion/sonar-project.properties + echo "sonar.pullrequest.base=origin/main" >> ingestion/sonar-project.properties + echo "sonar.pullrequest.provider=github" >> ingestion/sonar-project.properties + + make token=${{ secrets.INGESTION_SONAR_SECRET }} sonar_ingestion + + - name: Run Sonar + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: ${{ github.event_name == 'push' }} + run: make token=${{ secrets.INGESTION_SONAR_SECRET }} sonar_ingestion diff --git a/.gitignore b/.gitignore index 36ada895021..92389931405 100644 --- a/.gitignore +++ b/.gitignore @@ -84,4 +84,4 @@ ingestion/.python-version # MLFlow mlruns/ -/ingestion/tests/integration/mlflow/tests/db/ +/ingestion/tests/integration/source/mlflow/tests/db/ diff --git a/Makefile b/Makefile index 5cc0e026df3..022523468d6 100644 --- a/Makefile +++ b/Makefile @@ -11,10 +11,10 @@ install: python3 -m pip install ingestion/ install_test: - python3 -m pip install -r ingestion/requirements-test.txt + python3 -m pip install "ingestion[test]/" install_dev: - python3 -m pip install -r ingestion/requirements-dev.txt + python3 -m pip install "ingestion[dev]/" precommit_install: @echo "Installing pre-commit hooks" @@ -39,8 +39,25 @@ generate: datamodel-codegen --input catalog-rest-service/src/main/resources/json --input-file-type jsonschema --output ingestion/src/metadata/generated run_ometa_integration_tests: - cd ingestion; \ - pytest -c setup.cfg --override-ini=testpaths="tests/integration/ometa tests/unit/stage_test.py" + coverage run -m pytest -c ingestion/setup.cfg --doctest-modules --junitxml=ingestion/junit/test-results-integration.xml --override-ini=testpaths="ingestion/tests/integration/ometa ingestion/tests/integration/stage" + +unit_ingestion: + coverage run -m pytest -c ingestion/setup.cfg -s --doctest-modules --junitxml=ingestion/junit/test-results-unit.xml --override-ini=testpaths="ingestion/tests/unit" + +coverage: + coverage erase + make unit_ingestion + make run_ometa_integration_tests + coverage xml -i -o ingestion/coverage.xml + +sonar_ingestion: + docker run \ + --rm \ + -e SONAR_HOST_URL="https://sonarcloud.io" \ + -e SONAR_LOGIN=$(token) \ + -v ${PWD}:/usr/src \ + sonarsource/sonar-scanner-cli \ + -Dproject.settings=ingestion/sonar-project.properties publish: make install_dev generate diff --git a/ingestion/requirements-dev.txt b/ingestion/requirements-dev.txt deleted file mode 100644 index 6cbab24ad9b..00000000000 --- a/ingestion/requirements-dev.txt +++ /dev/null @@ -1,6 +0,0 @@ -boto3==1.20.14 -botocore==1.23.14 -datamodel-code-generator==0.11.14 -docker -google-cloud-storage==1.43.0 -twine diff --git a/ingestion/requirements-test.txt b/ingestion/requirements-test.txt deleted file mode 100644 index c64deea5b81..00000000000 --- a/ingestion/requirements-test.txt +++ /dev/null @@ -1,7 +0,0 @@ -black -isort -pre-commit -pylint -pytest -pytest-cov -faker \ No newline at end of file diff --git a/ingestion/setup.cfg b/ingestion/setup.cfg index 15c158b9056..ca280fe1780 100644 --- a/ingestion/setup.cfg +++ b/ingestion/setup.cfg @@ -33,7 +33,6 @@ indent=' ' sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER [tool:pytest] -addopts = --cov src --cov-report term --cov-config setup.cfg --strict-markers markers = slow: marks tests as slow (deselect with '-m "not slow"') testpaths = diff --git a/ingestion/setup.py b/ingestion/setup.py index 57a5543e40d..f571103f95c 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -108,6 +108,24 @@ plugins: Dict[str, Set[str]] = { "okta": {"okta~=2.3.0"}, "mlflow": {"mlflow-skinny~=1.22.0"}, } +dev = { + "boto3==1.20.14", + "botocore==1.23.14", + "datamodel-code-generator==0.11.14", + "docker", + "google-cloud-storage==1.43.0", + "twine", +} +test = { + "black", + "isort", + "pre-commit", + "pylint", + "pytest", + "pytest-cov", + "faker", + "coverage", +} build_options = {"includes": ["_cffi_backend"]} setup( @@ -138,6 +156,8 @@ setup( install_requires=list(base_requirements), extras_require={ "base": list(base_requirements), + "dev": list(dev), + "test": list(test), **{plugin: list(dependencies) for (plugin, dependencies) in plugins.items()}, "all": list( base_requirements.union( diff --git a/ingestion/sonar-project.properties b/ingestion/sonar-project.properties new file mode 100644 index 00000000000..240af043c1b --- /dev/null +++ b/ingestion/sonar-project.properties @@ -0,0 +1,11 @@ +sonar.projectKey=openmetadata-ingestion +sonar.projectName=openmetadata-ingestion +sonar.organization=open-metadata +sonar.language=py + +sonar.sources=ingestion/src +sonar.tests=ingestion/tests +sonar.exclusions=ingestion/src/metadata_server/static/**,ingestion/src/metadata_server/templates/** +sonar.python.xunit.reportPath=ingestion/junit/test-results-*.xml +sonar.python.coverage.reportPaths=ingestion/coverage.xml +sonar.python.version=3.8,3.9 diff --git a/ingestion/tests/integration/mlflow/__init__.py b/ingestion/tests/integration/ometa/__init__.py similarity index 100% rename from ingestion/tests/integration/mlflow/__init__.py rename to ingestion/tests/integration/ometa/__init__.py diff --git a/ingestion/tests/integration/source/__init__.py b/ingestion/tests/integration/source/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/integration/source/glue/__init__.py b/ingestion/tests/integration/source/glue/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/unit/glue_test.py b/ingestion/tests/integration/source/glue/glue_test.py similarity index 100% rename from ingestion/tests/unit/glue_test.py rename to ingestion/tests/integration/source/glue/glue_test.py diff --git a/ingestion/tests/integration/source/hive/__init__.py b/ingestion/tests/integration/source/hive/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/integration/hive/test_hive_crud.py b/ingestion/tests/integration/source/hive/test_hive_crud.py similarity index 100% rename from ingestion/tests/integration/hive/test_hive_crud.py rename to ingestion/tests/integration/source/hive/test_hive_crud.py diff --git a/ingestion/tests/integration/hive/tests/docker-compose.yml b/ingestion/tests/integration/source/hive/tests/docker-compose.yml similarity index 94% rename from ingestion/tests/integration/hive/tests/docker-compose.yml rename to ingestion/tests/integration/source/hive/tests/docker-compose.yml index a7ef6267519..de725e774d4 100644 --- a/ingestion/tests/integration/hive/tests/docker-compose.yml +++ b/ingestion/tests/integration/source/hive/tests/docker-compose.yml @@ -19,7 +19,7 @@ services: environment: - CLUSTER_NAME=test env_file: - - ./hadoop-hive.env + - hadoop-hive.env ports: - "50070:50070" datanode: @@ -27,7 +27,7 @@ services: volumes: - datanode:/hadoop/dfs/data env_file: - - ./hadoop-hive.env + - hadoop-hive.env environment: SERVICE_PRECONDITION: "namenode:50070" ports: @@ -35,7 +35,7 @@ services: hive-server: image: bde2020/hive:2.3.2-postgresql-metastore env_file: - - ./hadoop-hive.env + - hadoop-hive.env command: "sh /setup/setup.sh" volumes: - ./setup:/setup @@ -47,7 +47,7 @@ services: hive-metastore: image: bde2020/hive:2.3.2-postgresql-metastore env_file: - - ./hadoop-hive.env + - hadoop-hive.env command: /opt/hive/bin/hive --service metastore environment: SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432" diff --git a/ingestion/tests/integration/hive/tests/hadoop-hive.env b/ingestion/tests/integration/source/hive/tests/hadoop-hive.env similarity index 100% rename from ingestion/tests/integration/hive/tests/hadoop-hive.env rename to ingestion/tests/integration/source/hive/tests/hadoop-hive.env diff --git a/ingestion/tests/integration/hive/tests/setup/hive_setup.sql b/ingestion/tests/integration/source/hive/tests/setup/hive_setup.sql similarity index 100% rename from ingestion/tests/integration/hive/tests/setup/hive_setup.sql rename to ingestion/tests/integration/source/hive/tests/setup/hive_setup.sql diff --git a/ingestion/tests/integration/hive/tests/setup/setup.sh b/ingestion/tests/integration/source/hive/tests/setup/setup.sh similarity index 100% rename from ingestion/tests/integration/hive/tests/setup/setup.sh rename to ingestion/tests/integration/source/hive/tests/setup/setup.sh diff --git a/ingestion/tests/integration/source/ldap/__init__.py b/ingestion/tests/integration/source/ldap/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/integration/ldap/test_ldap_crud.py b/ingestion/tests/integration/source/ldap/test_ldap_crud.py similarity index 100% rename from ingestion/tests/integration/ldap/test_ldap_crud.py rename to ingestion/tests/integration/source/ldap/test_ldap_crud.py diff --git a/ingestion/tests/integration/ldap/tests/bootstrap.ldif b/ingestion/tests/integration/source/ldap/tests/bootstrap.ldif similarity index 100% rename from ingestion/tests/integration/ldap/tests/bootstrap.ldif rename to ingestion/tests/integration/source/ldap/tests/bootstrap.ldif diff --git a/ingestion/tests/integration/ldap/tests/docker-compose.yml b/ingestion/tests/integration/source/ldap/tests/docker-compose.yml similarity index 100% rename from ingestion/tests/integration/ldap/tests/docker-compose.yml rename to ingestion/tests/integration/source/ldap/tests/docker-compose.yml diff --git a/ingestion/tests/integration/mlflow/Dockerfile b/ingestion/tests/integration/source/mlflow/Dockerfile similarity index 100% rename from ingestion/tests/integration/mlflow/Dockerfile rename to ingestion/tests/integration/source/mlflow/Dockerfile diff --git a/ingestion/tests/integration/mlflow/Makefile b/ingestion/tests/integration/source/mlflow/Makefile similarity index 100% rename from ingestion/tests/integration/mlflow/Makefile rename to ingestion/tests/integration/source/mlflow/Makefile diff --git a/ingestion/tests/integration/mlflow/README.md b/ingestion/tests/integration/source/mlflow/README.md similarity index 100% rename from ingestion/tests/integration/mlflow/README.md rename to ingestion/tests/integration/source/mlflow/README.md diff --git a/ingestion/tests/integration/source/mlflow/__init__.py b/ingestion/tests/integration/source/mlflow/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/integration/mlflow/docker-compose.yml b/ingestion/tests/integration/source/mlflow/docker-compose.yml similarity index 100% rename from ingestion/tests/integration/mlflow/docker-compose.yml rename to ingestion/tests/integration/source/mlflow/docker-compose.yml diff --git a/ingestion/tests/integration/mlflow/experiment.py b/ingestion/tests/integration/source/mlflow/experiment.py similarity index 100% rename from ingestion/tests/integration/mlflow/experiment.py rename to ingestion/tests/integration/source/mlflow/experiment.py diff --git a/ingestion/tests/integration/mlflow/train.py b/ingestion/tests/integration/source/mlflow/train.py similarity index 100% rename from ingestion/tests/integration/mlflow/train.py rename to ingestion/tests/integration/source/mlflow/train.py diff --git a/ingestion/tests/integration/source/mssql/__init__.py b/ingestion/tests/integration/source/mssql/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/integration/mssql/test_mssql_crud.py b/ingestion/tests/integration/source/mssql/test_mssql_crud.py similarity index 100% rename from ingestion/tests/integration/mssql/test_mssql_crud.py rename to ingestion/tests/integration/source/mssql/test_mssql_crud.py diff --git a/ingestion/tests/integration/mssql/tests/docker-compose.yml b/ingestion/tests/integration/source/mssql/tests/docker-compose.yml similarity index 100% rename from ingestion/tests/integration/mssql/tests/docker-compose.yml rename to ingestion/tests/integration/source/mssql/tests/docker-compose.yml diff --git a/ingestion/tests/integration/mssql/tests/setup/setup.sh b/ingestion/tests/integration/source/mssql/tests/setup/setup.sh similarity index 100% rename from ingestion/tests/integration/mssql/tests/setup/setup.sh rename to ingestion/tests/integration/source/mssql/tests/setup/setup.sh diff --git a/ingestion/tests/integration/mssql/tests/setup/setup.sql b/ingestion/tests/integration/source/mssql/tests/setup/setup.sql similarity index 100% rename from ingestion/tests/integration/mssql/tests/setup/setup.sql rename to ingestion/tests/integration/source/mssql/tests/setup/setup.sql diff --git a/ingestion/tests/integration/source/mysql/__init__.py b/ingestion/tests/integration/source/mysql/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/integration/mysql/test_mysql_crud.py b/ingestion/tests/integration/source/mysql/test_mysql_crud.py similarity index 100% rename from ingestion/tests/integration/mysql/test_mysql_crud.py rename to ingestion/tests/integration/source/mysql/test_mysql_crud.py diff --git a/ingestion/tests/integration/mysql/tests/docker-compose.yml b/ingestion/tests/integration/source/mysql/tests/docker-compose.yml similarity index 100% rename from ingestion/tests/integration/mysql/tests/docker-compose.yml rename to ingestion/tests/integration/source/mysql/tests/docker-compose.yml diff --git a/ingestion/tests/integration/mysql/tests/setup/setup.sql b/ingestion/tests/integration/source/mysql/tests/setup/setup.sql similarity index 100% rename from ingestion/tests/integration/mysql/tests/setup/setup.sql rename to ingestion/tests/integration/source/mysql/tests/setup/setup.sql diff --git a/ingestion/tests/integration/source/postgres/__init__.py b/ingestion/tests/integration/source/postgres/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/integration/postgres/test_postgres_crud.py b/ingestion/tests/integration/source/postgres/test_postgres_crud.py similarity index 100% rename from ingestion/tests/integration/postgres/test_postgres_crud.py rename to ingestion/tests/integration/source/postgres/test_postgres_crud.py diff --git a/ingestion/tests/integration/postgres/tests/docker-compose.yml b/ingestion/tests/integration/source/postgres/tests/docker-compose.yml similarity index 100% rename from ingestion/tests/integration/postgres/tests/docker-compose.yml rename to ingestion/tests/integration/source/postgres/tests/docker-compose.yml diff --git a/ingestion/tests/integration/source/trino/__init__.py b/ingestion/tests/integration/source/trino/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/integration/trino/test_trino_crud.py b/ingestion/tests/integration/source/trino/test_trino_crud.py similarity index 100% rename from ingestion/tests/integration/trino/test_trino_crud.py rename to ingestion/tests/integration/source/trino/test_trino_crud.py diff --git a/ingestion/tests/integration/trino/tests/docker-compose.yml b/ingestion/tests/integration/source/trino/tests/docker-compose.yml similarity index 100% rename from ingestion/tests/integration/trino/tests/docker-compose.yml rename to ingestion/tests/integration/source/trino/tests/docker-compose.yml diff --git a/ingestion/tests/unit/trino_test.py b/ingestion/tests/integration/source/trino/trino_test.py similarity index 100% rename from ingestion/tests/unit/trino_test.py rename to ingestion/tests/integration/source/trino/trino_test.py diff --git a/ingestion/tests/integration/stage/__init__.py b/ingestion/tests/integration/stage/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/unit/stage_test.py b/ingestion/tests/integration/stage/stage_test.py similarity index 93% rename from ingestion/tests/unit/stage_test.py rename to ingestion/tests/integration/stage/stage_test.py index c22ffdb448c..1f3bb82910b 100644 --- a/ingestion/tests/unit/stage_test.py +++ b/ingestion/tests/integration/stage/stage_test.py @@ -8,7 +8,7 @@ config = """ "source": { "type": "sample-data", "config": { - "sample_data_folder": "examples/sample_data/" + "sample_data_folder": "ingestion/examples/sample_data" } }, "stage": { diff --git a/ingestion/tests/integration/workflow/__init__.py b/ingestion/tests/integration/workflow/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ingestion/tests/unit/mysql_test.json b/ingestion/tests/integration/workflow/mysql_test.json similarity index 100% rename from ingestion/tests/unit/mysql_test.json rename to ingestion/tests/integration/workflow/mysql_test.json diff --git a/ingestion/tests/unit/workflow_test.py b/ingestion/tests/integration/workflow/workflow_test.py similarity index 100% rename from ingestion/tests/unit/workflow_test.py rename to ingestion/tests/integration/workflow/workflow_test.py diff --git a/ingestion/tests/unit/test_config.py b/ingestion/tests/unit/test_config.py index cff8826db73..39e6f7280d9 100644 --- a/ingestion/tests/unit/test_config.py +++ b/ingestion/tests/unit/test_config.py @@ -24,7 +24,7 @@ class TestConfig(TestCase): Check config reading """ - basedir = os.path.join("resources", "config") + basedir = os.path.join(os.path.dirname(__file__), "resources", "config") def test_basic(self): """