diff --git a/ingestion/tests/integration/workflow/mysql_test.json b/ingestion/tests/integration/workflow/mysql_test.json index ad75fc11125..533868aeae4 100644 --- a/ingestion/tests/integration/workflow/mysql_test.json +++ b/ingestion/tests/integration/workflow/mysql_test.json @@ -1,27 +1,32 @@ { "source": { "type": "mysql", - "config": { - "username": "openmetadata_user", - "password": "openmetadata_password", - "database": "openmetadata_db", - "service_name": "local_mysql_test", - "schema_filter_pattern": { - "excludes": ["mysql.*", "information_schema.*", "performance_schema.*", "sys.*"] + "serviceName": "local_mysql_test", + "serviceConnection": { + "config": { + "type": "Mysql", + "username": "openmetadata_user", + "database":"openmetadata_db", + "password": "openmetadata_password", + "hostPort": "localhost:3306", + "connectionOptions": {}, + "connectionArguments": {} + } + }, + "sourceConfig": { + "config": { + "enableDataProfiler": false } } }, - "stage": { - "type": "file", - "config": { - "filename": "/tmp/mysql_test" - } + "sink": { + "type": "metadata-rest", + "config": {} }, - "metadata_server": { - "type": "metadata-server", - "config": { - "api_endpoint": "http://localhost:8585/api", - "auth_provider_type": "no-auth" + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "no-auth" } } -} +} \ No newline at end of file diff --git a/ingestion/tests/integration/workflow/mysql_test.yaml b/ingestion/tests/integration/workflow/mysql_test.yaml new file mode 100644 index 00000000000..95dda20e558 --- /dev/null +++ b/ingestion/tests/integration/workflow/mysql_test.yaml @@ -0,0 +1,28 @@ +source: + type: mysql + serviceName: local_mysql_test + serviceConnection: + config: + type: Mysql + username: openmetadata_user + password: openmetadata_password + database: openmetadata_db + hostPort: localhost:3306 + connectionOptions: {} + connectionArguments: {} + sourceConfig: + config: + enableDataProfiler: false + schemaFilterPattern: + excludes: + - mysql.* + - information_schema.* + - performance_schema.* + - sys.* +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: no-auth diff --git a/ingestion/tests/integration/workflow/workflow_test.py b/ingestion/tests/integration/workflow/workflow_test.py index 0a21d3a429b..d2aa63b12bc 100644 --- a/ingestion/tests/integration/workflow/workflow_test.py +++ b/ingestion/tests/integration/workflow/workflow_test.py @@ -14,6 +14,9 @@ import pathlib from unittest import TestCase from metadata.config.common import ConfigurationError, load_config_file +from metadata.generated.schema.metadataIngestion.workflow import ( + OpenMetadataWorkflowConfig, +) from metadata.ingestion.api.workflow import Workflow from metadata.ingestion.ometa.ometa_api import OpenMetadata @@ -27,12 +30,11 @@ class WorkflowTest(TestCase): self.assertEqual((my_class is not None), True) def test_get_4xx(self): - my_class = None key = "metadata.ingestion.sink.MYSQL.mysqlSINK" try: if key.find(".") >= 0: module_name, class_name = key.rsplit(".", 1) - my_class = getattr(importlib.import_module(module_name), class_name) + getattr(importlib.import_module(module_name), class_name) except ModuleNotFoundError: self.assertRaises(ModuleNotFoundError) @@ -60,19 +62,18 @@ class WorkflowTest(TestCase): def test_execute_200(self): current_dir = pathlib.Path(__file__).resolve().parent - config_file = current_dir.joinpath("mysql_test.json") + config_file = current_dir.joinpath("mysql_test.yaml") workflow_config = load_config_file(config_file) workflow = Workflow.create(workflow_config) workflow.execute() workflow.stop() - config = MetadataServerConfig.parse_obj( - workflow_config.get("metadata_server").get("config") - ) + config = workflow.config.workflowConfig.openMetadataServerConfig client = OpenMetadata(config).client client.delete( f"/services/databaseServices/" f"{client.get('/services/databaseServices/name/local_mysql_test')['id']}" + f"?hardDelete=true&recursive=true" ) file_path = "/tmp/mysql_test" with open(file_path) as ingestionFile: @@ -82,6 +83,6 @@ class WorkflowTest(TestCase): def test_execute_4xx(self): config_file = pathlib.Path("/tmp/mysql_test123") try: - workflow_config = load_config_file(config_file) + load_config_file(config_file) except ConfigurationError: self.assertRaises(ConfigurationError)