From 9edd824c3ad83f01d83f00bdb0e88a650ca95f14 Mon Sep 17 00:00:00 2001 From: vijaypm Date: Sat, 23 Oct 2021 19:58:26 -0700 Subject: [PATCH] Issue 910 (#914) * Issue 898 (#905) * ISSUE-898: additional information in the prerequisities for building and running code * ISSUE-898: removed unreachable old doc * ISSUE-898: added new docker compose to expose MySQL and ES ports to host machines * ISSUE-898: changed jdbc connect url to allow Public Key Retrieval * ISSUE-898: fixed log name to openmetadata.log Co-authored-by: Vijay Mariadassou * Fixes #906 Remove unused methods lingering from #899 * Update pull_request_template.md * Update pull_request_template.md * ISSUE-861: add elasticsearch username & password (#894) * ISSUE-861: add elasticsearch username & password * ISSUE-861: python elasticsearch sink add username & password * ISSUE-861: bugfix * format code * format code * updated instructions to run integration tests * fixed api call to metadata server; changed test to cover both database as well as table operations everytime Co-authored-by: Vijay Mariadassou Co-authored-by: sureshms Co-authored-by: Suresh Srinivas Co-authored-by: rong fengliang <1141591465@qq.com> --- .../developer/run-integration-tests.md | 13 +++++--- .../integration/mysql/test_mysql_crud.py | 31 +++++++++++-------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/docs/open-source-community/developer/run-integration-tests.md b/docs/open-source-community/developer/run-integration-tests.md index fd885069585..9af81fb010a 100644 --- a/docs/open-source-community/developer/run-integration-tests.md +++ b/docs/open-source-community/developer/run-integration-tests.md @@ -1,15 +1,20 @@ # Run Integration Tests {% hint style="info" %} -Make sure OpenMetadata is up and running +Make sure OpenMetadata is up and running. Refer to instructions [building and running](build-code-run-tests.md). {% endhint %} ## Run MySQL test +Run the following commands from the top-level directory ```text -cd ingestion -source env/bin/activate -cd tests/integration/mysql +python3 -m venv /tmp/venv +source /tmp/venv/bin/activate +pip install -r ingestion/requirements.txt +pip install -e ingestion +pip install pytest +pip install pytest-docker +cd ingestion/tests/integration/mysql pytest -s -c /dev/null ``` diff --git a/ingestion/tests/integration/mysql/test_mysql_crud.py b/ingestion/tests/integration/mysql/test_mysql_crud.py index bdb0a935989..afc1dfa3927 100644 --- a/ingestion/tests/integration/mysql/test_mysql_crud.py +++ b/ingestion/tests/integration/mysql/test_mysql_crud.py @@ -14,7 +14,6 @@ # limitations under the License. import time -from metadata.ingestion.ometa.client import REST from metadata.generated.schema.type.entityReference import EntityReference from metadata.generated.schema.entity.data.table import Column from metadata.generated.schema.api.services.createDatabaseService import CreateDatabaseServiceEntityRequest @@ -27,6 +26,8 @@ from requests.exceptions import ConnectionError from sqlalchemy.engine import create_engine from sqlalchemy.inspection import inspect +from metadata.ingestion.ometa.openmetadata_rest import MetadataServerConfig, OpenMetadataAPIClient + def is_responsive(url): try: @@ -39,8 +40,8 @@ def is_responsive(url): def create_delete_table(client): databases = client.list_databases() - columns = [Column(name="id", columnDataType="INT"), - Column(name="name", columnDataType="VARCHAR")] + columns = [Column(name="id", dataType="INT", dataLength=1), + Column(name="name", dataType="VARCHAR", dataLength=1)] table = CreateTableEntityRequest( name="test1", columns=columns, database=databases[0].id) created_table = client.create_or_update_table(table) @@ -57,7 +58,7 @@ def create_delete_table(client): def create_delete_database(client): data = {'jdbc': {'connectionUrl': 'mysql://localhost/catalog_db', 'driverClass': 'jdbc'}, 'name': 'temp_local_mysql', - 'serviceType': 'MYSQL', + 'serviceType': 'MySQL', 'description': 'local mysql env'} create_mysql_service = CreateDatabaseServiceEntityRequest(**data) mysql_service = client.create_database_service(create_mysql_service) @@ -78,25 +79,29 @@ def catalog_service(docker_ip, docker_services): port = docker_services.port_for("db", 3306) print("Mysql is running on port {}".format(port)) url = "http://localhost:8585" - time.sleep(420) + time.sleep(30) docker_services.wait_until_responsive( - timeout=60.0, pause=0.5, check=lambda: is_responsive(url) + timeout=30.0, pause=0.5, check=lambda: is_responsive(url) ) return url def test_check_tables(catalog_service): - client = REST(catalog_service + "/api", 'test', 'test') + metadata_config = MetadataServerConfig.parse_obj( + { + "api_endpoint": catalog_service + "/api", + "auth_provider_type": "no-auth" + } + ) + client = OpenMetadataAPIClient(metadata_config) databases = client.list_databases() - if len(databases) > 0: - assert create_delete_table(client) - else: - assert create_delete_database(client) - + assert create_delete_database(client) def test_read_schema(): url = "mysql+pymysql://catalog_user:catalog_password@localhost:3307" - engine = create_engine(url) + # pool_recycle to avoid the occasional "Lost connection to MySQL server during query" error + # when host machine is slow + engine = create_engine(url, pool_recycle=1) inspector = inspect(engine) schemas = [] for schema in inspector.get_schema_names():