Fix #4758: Add support for yaml configuration in Ingestion connectors (#4759)

* Fix #4758: Add support for yaml configuration in Ingestion connectors

* Fix #4758: Add support for yaml configuration in Ingestion connectors

* Fix #4758: Add support for yaml configuration in Ingestion connectors

* Fix #4758: Add support for yaml configuration in Ingestion connectors

ingestion/pipelines/sample_data.yaml

@@ -0,0 +1,16 @@
+---
+source:
+  type: sample-data
+  serviceName: sample_data
+  serviceConnection:
+    config:
+      type: SampleData
+      sampleDataFolder: "./examples/sample_data"
+  sourceConfig: {}
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: no-auth

ingestion/src/metadata/config/common.py

@@ -16,6 +16,7 @@ import pathlib
 from abc import ABC, abstractmethod
 from typing import IO, Any, Optional
 
+import yaml
 from pydantic import BaseModel
 
 FQDN_SEPARATOR: str = "."
@@ -45,22 +46,40 @@ class ConfigurationMechanism(ABC):
         pass
 
 
+class YamlConfigurationMechanism(ConfigurationMechanism):
+    """load configuration from yaml files"""
+
+    def load_config(self, config_fp: IO) -> dict:
+        config = yaml.safe_load(config_fp)
+        return config
+
+
+class JsonConfigurationMechanism(ConfigurationMechanism):
+    """load configuration from json files"""
+
+    def load_config(self, config_fp: IO) -> dict:
+        config = json.load(config_fp)
+        return config
+
+
 def load_config_file(config_file: pathlib.Path) -> dict:
     if not config_file.is_file():
         raise ConfigurationError(f"Cannot open config file {config_file}")
 
-    if config_file.suffix not in [".json"]:
+    config_mech: ConfigurationMechanism
+    if config_file.suffix in [".yaml", ".yml"]:
+        config_mech = YamlConfigurationMechanism()
+    elif config_file.suffix == ".json":
+        config_mech = JsonConfigurationMechanism()
+    else:
         raise ConfigurationError(
-            "Only json are supported. Cannot process file type {}".format(
+            "Only .json and .yml are supported. Cannot process file type {}".format(
                 config_file.suffix
             )
         )
-
     with config_file.open() as raw_config_file:
         raw_config = raw_config_file.read()
-
     expanded_config_file = os.path.expandvars(raw_config)
     config_fp = io.StringIO(expanded_config_file)
-    config = json.load(config_fp)
+    config = config_mech.load_config(config_fp)
-
     return config