diff --git a/ingestion/pipelines/sample_data.yaml b/ingestion/pipelines/sample_data.yaml new file mode 100644 index 00000000000..7369b650820 --- /dev/null +++ b/ingestion/pipelines/sample_data.yaml @@ -0,0 +1,16 @@ +--- +source: + type: sample-data + serviceName: sample_data + serviceConnection: + config: + type: SampleData + sampleDataFolder: "./examples/sample_data" + sourceConfig: {} +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: no-auth diff --git a/ingestion/src/metadata/config/common.py b/ingestion/src/metadata/config/common.py index a7cb847ab13..8ef5db1cbc7 100644 --- a/ingestion/src/metadata/config/common.py +++ b/ingestion/src/metadata/config/common.py @@ -16,6 +16,7 @@ import pathlib from abc import ABC, abstractmethod from typing import IO, Any, Optional +import yaml from pydantic import BaseModel FQDN_SEPARATOR: str = "." @@ -45,22 +46,40 @@ class ConfigurationMechanism(ABC): pass +class YamlConfigurationMechanism(ConfigurationMechanism): + """load configuration from yaml files""" + + def load_config(self, config_fp: IO) -> dict: + config = yaml.safe_load(config_fp) + return config + + +class JsonConfigurationMechanism(ConfigurationMechanism): + """load configuration from json files""" + + def load_config(self, config_fp: IO) -> dict: + config = json.load(config_fp) + return config + + def load_config_file(config_file: pathlib.Path) -> dict: if not config_file.is_file(): raise ConfigurationError(f"Cannot open config file {config_file}") - if config_file.suffix not in [".json"]: + config_mech: ConfigurationMechanism + if config_file.suffix in [".yaml", ".yml"]: + config_mech = YamlConfigurationMechanism() + elif config_file.suffix == ".json": + config_mech = JsonConfigurationMechanism() + else: raise ConfigurationError( - "Only json are supported. Cannot process file type {}".format( + "Only .json and .yml are supported. Cannot process file type {}".format( config_file.suffix ) ) - with config_file.open() as raw_config_file: raw_config = raw_config_file.read() - expanded_config_file = os.path.expandvars(raw_config) config_fp = io.StringIO(expanded_config_file) - config = json.load(config_fp) - + config = config_mech.load_config(config_fp) return config