yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-01 19:25:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(ingest) - remove snowflake_common dependency on aws_common (#4054)

Browse Source 
Co-authored-by: Shirshanka Das <shirshanka@apache.org>
This commit is contained in:
Aditya Radhakrishnan 2022-02-04 23:24:20 -08:00 committed by GitHub
parent 1a2f75b1e6
commit 61db4ed152
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 22 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py
View File

@ -102,19 +102,3 @@ class AwsSourceConfig(ConfigModel):
def get_sagemaker_client(self) -> "SageMakerClient":
return self.get_session().client("sagemaker")
def make_s3_urn(s3_uri: str, env: str, suffix: Optional[str] = None) -> str:
if not s3_uri.startswith("s3://"):
raise ValueError("S3 URIs should begin with 's3://'")
# remove S3 prefix (s3://)
s3_name = s3_uri[5:]
if s3_name.endswith("/"):
s3_name = s3_name[:-1]
if suffix is not None:
return f"urn:li:dataset:(urn:li:dataPlatform:s3,{s3_name}_{suffix},{env})"
return f"urn:li:dataset:(urn:li:dataPlatform:s3,{s3_name},{env})"

3
metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
View File

@ -15,7 +15,8 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper
from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.source import Source, SourceReport
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.aws.aws_common import AwsSourceConfig, make_s3_urn
from datahub.ingestion.source.aws.aws_common import AwsSourceConfig
from datahub.ingestion.source.aws.s3_util import make_s3_urn
from datahub.metadata.com.linkedin.pegasus2avro.common import Status
from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent

17
metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py Normal file
View File

@ -0,0 +1,17 @@
from typing import Optional
def make_s3_urn(s3_uri: str, env: str, suffix: Optional[str] = None) -> str:
if not s3_uri.startswith("s3://"):
raise ValueError("S3 URIs should begin with 's3://'")
# remove S3 prefix (s3://)
s3_name = s3_uri[5:]
if s3_name.endswith("/"):
s3_name = s3_name[:-1]
if suffix is not None:
return f"urn:li:dataset:(urn:li:dataPlatform:s3,{s3_name}_{suffix},{env})"
return f"urn:li:dataset:(urn:li:dataPlatform:s3,{s3_name},{env})"

2
metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/jobs.py
View File

@ -19,7 +19,7 @@ from typing import (
from datahub.emitter import mce_builder
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.aws.aws_common import make_s3_urn
from datahub.ingestion.source.aws.s3_util import make_s3_urn
from datahub.ingestion.source.aws.sagemaker_processors.common import (
SagemakerSourceReport,
)

2
metadata-ingestion/src/datahub/ingestion/source/data_lake/__init__.py
View File

@ -35,7 +35,7 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper
from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.source import Source, SourceReport
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.aws.aws_common import make_s3_urn
from datahub.ingestion.source.aws.s3_util import make_s3_urn
from datahub.ingestion.source.data_lake.config import DataLakeSourceConfig
from datahub.ingestion.source.data_lake.profiling import _SingleTableProfiler
from datahub.ingestion.source.data_lake.report import DataLakeSourceReport

2
metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py
View File

@ -24,7 +24,7 @@ from datahub.configuration.common import AllowDenyPattern
from datahub.configuration.time_window_config import BaseTimeWindowConfig
from datahub.emitter.mcp import MetadataChangeProposalWrapper
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.aws.aws_common import make_s3_urn
from datahub.ingestion.source.aws.s3_util import make_s3_urn
from datahub.ingestion.source.sql.sql_common import (
RecordTypeClass,
SQLAlchemyConfig,