issue-3828: updated-workflow-and-stage-test (#5117)

ingestion/tests/integration/workflow/mysql_test.json

@@ -1,27 +1,32 @@
 {
   "source": {
     "type": "mysql",
-    "config": {
-      "username": "openmetadata_user",
-      "password": "openmetadata_password",
-      "database": "openmetadata_db",
-      "service_name": "local_mysql_test",
-      "schema_filter_pattern": {
-        "excludes": ["mysql.*", "information_schema.*", "performance_schema.*", "sys.*"]
+    "serviceName": "local_mysql_test",
+    "serviceConnection": {
+      "config": {
+        "type": "Mysql",
+        "username": "openmetadata_user",
+        "database":"openmetadata_db",
+        "password": "openmetadata_password",
+        "hostPort": "localhost:3306",
+        "connectionOptions": {},
+        "connectionArguments": {}
+      }
+    },
+    "sourceConfig": {
+      "config": {
+        "enableDataProfiler": false
       }
     }
   },
-  "stage": {
-    "type": "file",
-    "config": {
-      "filename": "/tmp/mysql_test"
-    }
+  "sink": {
+    "type": "metadata-rest",
+    "config": {}
   },
-  "metadata_server": {
-    "type": "metadata-server",
-    "config": {
-      "api_endpoint": "http://localhost:8585/api",
-      "auth_provider_type": "no-auth"
+  "workflowConfig": {
+    "openMetadataServerConfig": {
+      "hostPort": "http://localhost:8585/api",
+      "authProvider": "no-auth"
     }
   }
-}
+}

ingestion/tests/integration/workflow/mysql_test.yaml

@@ -0,0 +1,28 @@
+source:
+  type: mysql
+  serviceName: local_mysql_test
+  serviceConnection:
+    config:
+      type: Mysql
+      username: openmetadata_user
+      password: openmetadata_password
+      database: openmetadata_db
+      hostPort: localhost:3306
+      connectionOptions: {}
+      connectionArguments: {}
+  sourceConfig:
+    config:
+      enableDataProfiler: false
+      schemaFilterPattern:
+        excludes:
+        - mysql.*
+        - information_schema.*
+        - performance_schema.*
+        - sys.*
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: no-auth

ingestion/tests/integration/workflow/workflow_test.py

@@ -14,6 +14,9 @@ import pathlib
 from unittest import TestCase
 
 from metadata.config.common import ConfigurationError, load_config_file
+from metadata.generated.schema.metadataIngestion.workflow import (
+    OpenMetadataWorkflowConfig,
+)
 from metadata.ingestion.api.workflow import Workflow
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 
@@ -27,12 +30,11 @@ class WorkflowTest(TestCase):
             self.assertEqual((my_class is not None), True)
 
     def test_get_4xx(self):
-        my_class = None
         key = "metadata.ingestion.sink.MYSQL.mysqlSINK"
         try:
             if key.find(".") >= 0:
                 module_name, class_name = key.rsplit(".", 1)
-                my_class = getattr(importlib.import_module(module_name), class_name)
+                getattr(importlib.import_module(module_name), class_name)
         except ModuleNotFoundError:
             self.assertRaises(ModuleNotFoundError)
 
@@ -60,19 +62,18 @@ class WorkflowTest(TestCase):
 
     def test_execute_200(self):
         current_dir = pathlib.Path(__file__).resolve().parent
-        config_file = current_dir.joinpath("mysql_test.json")
+        config_file = current_dir.joinpath("mysql_test.yaml")
         workflow_config = load_config_file(config_file)
         workflow = Workflow.create(workflow_config)
         workflow.execute()
         workflow.stop()
-        config = MetadataServerConfig.parse_obj(
-            workflow_config.get("metadata_server").get("config")
-        )
+        config = workflow.config.workflowConfig.openMetadataServerConfig
         client = OpenMetadata(config).client
 
         client.delete(
             f"/services/databaseServices/"
             f"{client.get('/services/databaseServices/name/local_mysql_test')['id']}"
+            f"?hardDelete=true&recursive=true"
         )
         file_path = "/tmp/mysql_test"
         with open(file_path) as ingestionFile:
@@ -82,6 +83,6 @@ class WorkflowTest(TestCase):
     def test_execute_4xx(self):
         config_file = pathlib.Path("/tmp/mysql_test123")
         try:
-            workflow_config = load_config_file(config_file)
+            load_config_file(config_file)
         except ConfigurationError:
             self.assertRaises(ConfigurationError)