yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-09 07:53:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/demo-data): fix bug in path type (#7749)

Browse Source
This commit is contained in:
Harshal Sheth 2023-04-05 11:46:15 +05:30 committed by GitHub
parent 20504aae70
commit e06117af66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
metadata-ingestion/src/datahub/ingestion/source/demo_data.py
View File

@ -1,11 +1,14 @@
from typing import Iterable
from datahub.configuration.common import ConfigModel from datahub.configuration.common import ConfigModel
from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.common import PipelineContext, WorkUnit
from datahub.ingestion.api.decorators import ( from datahub.ingestion.api.decorators import (
SupportStatus, SupportStatus,
config_class, config_class,
platform_name, platform_name,
support_status, support_status,
) )
from datahub.ingestion.api.source import Source, SourceReport
from datahub.ingestion.source.file import FileSourceConfig, GenericFileSource from datahub.ingestion.source.file import FileSourceConfig, GenericFileSource
from datahub.utilities.sample_data import download_sample_data from datahub.utilities.sample_data import download_sample_data
@ -16,18 +19,19 @@ class DemoDataConfig(ConfigModel):
@platform_name("Demo Data") @platform_name("Demo Data")
@config_class(ConfigModel) @config_class(DemoDataConfig)
@support_status(SupportStatus.UNKNOWN) @support_status(SupportStatus.UNKNOWN)
class DemoDataSource(GenericFileSource): class DemoDataSource(Source):
""" """
This source loads sample data into DataHub. It is intended for demo and testing purposes only. This source loads sample data into DataHub. It is intended for demo and testing purposes only.
""" """
def __init__(self, ctx: PipelineContext, config: DemoDataConfig): def __init__(self, ctx: PipelineContext, config: DemoDataConfig):
file_config = FileSourceConfig(path=download_sample_data()) file_config = FileSourceConfig(path=str(download_sample_data()))
super().__init__(ctx, file_config) self.file_source = GenericFileSource(ctx, file_config)
@classmethod def get_workunits(self) -> Iterable[WorkUnit]:
def create(cls, config_dict, ctx): yield from self.file_source.get_workunits()
config = DemoDataConfig.parse_obj(config_dict or {})
return cls(ctx, config) def get_report(self) -> SourceReport:
return self.file_source.get_report()