yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-01 11:19:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/demo-data): fix bug in path type (#7749)

Browse Source
This commit is contained in:
Harshal Sheth 2023-04-05 11:46:15 +05:30 committed by GitHub
parent 20504aae70
commit e06117af66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
metadata-ingestion/src/datahub/ingestion/source/demo_data.py
View File

@ -1,11 +1,14 @@
from typing import Iterable
from datahub.configuration.common import ConfigModel
from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.common import PipelineContext, WorkUnit
from datahub.ingestion.api.decorators import (
SupportStatus,
config_class,
platform_name,
support_status,
)
from datahub.ingestion.api.source import Source, SourceReport
from datahub.ingestion.source.file import FileSourceConfig, GenericFileSource
from datahub.utilities.sample_data import download_sample_data
@ -16,18 +19,19 @@ class DemoDataConfig(ConfigModel):
@platform_name("Demo Data")
@config_class(ConfigModel)
@config_class(DemoDataConfig)
@support_status(SupportStatus.UNKNOWN)
class DemoDataSource(GenericFileSource):
class DemoDataSource(Source):
"""
This source loads sample data into DataHub. It is intended for demo and testing purposes only.
"""
def __init__(self, ctx: PipelineContext, config: DemoDataConfig):
file_config = FileSourceConfig(path=download_sample_data())
super().__init__(ctx, file_config)
file_config = FileSourceConfig(path=str(download_sample_data()))
self.file_source = GenericFileSource(ctx, file_config)
@classmethod
def create(cls, config_dict, ctx):
config = DemoDataConfig.parse_obj(config_dict or {})
return cls(ctx, config)
def get_workunits(self) -> Iterable[WorkUnit]:
yield from self.file_source.get_workunits()
def get_report(self) -> SourceReport:
return self.file_source.get_report()