yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-14 11:41:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/airflow): able to set platform_instance in Dataset (#8313)

Browse Source 
Co-authored-by: Andrew Sikowitz <andrew.sikowitz@acryl.io>
This commit is contained in:
Đặng Minh Dũng 2023-07-13 06:37:36 +07:00 committed by GitHub
parent 61a1f5775c
commit 64bcc132a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
metadata-ingestion/src/datahub_provider/entities.py
View File

@ -1,4 +1,5 @@
from abc import abstractmethod
from typing import Optional
import attr
@ -18,10 +19,16 @@ class Dataset(_Entity):
platform: str
name: str
env: str = builder.DEFAULT_ENV
platform_instance: Optional[str] = None
@property
def urn(self):
return builder.make_dataset_urn(self.platform, self.name, self.env)
return builder.make_dataset_urn_with_platform_instance(
platform=self.platform,
name=self.name,
platform_instance=self.platform_instance,
env=self.env,
)
@attr.s(str=True)

9
metadata-ingestion/src/datahub_provider/example_dags/lineage_backend_demo.py
View File

@ -34,8 +34,13 @@ with DAG(
dag=dag,
bash_command="echo 'This is where you might run your data tooling.'",
inlets=[
Dataset("snowflake", "mydb.schema.tableA"),
Dataset("snowflake", "mydb.schema.tableB", "DEV"),
Dataset(platform="snowflake", name="mydb.schema.tableA"),
Dataset(platform="snowflake", name="mydb.schema.tableB", env="DEV"),
Dataset(
platform="snowflake",
name="mydb.schema.tableC",
platform_instance="cloud",
),
# You can also put dataset URNs in the inlets/outlets lists.
Urn(
"urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)"

12
metadata-ingestion/src/datahub_provider/example_dags/lineage_emission_dag.py
View File

@ -57,11 +57,17 @@ with DAG(
mces=[
builder.make_lineage_mce(
upstream_urns=[
builder.make_dataset_urn("snowflake", "mydb.schema.tableA"),
builder.make_dataset_urn("snowflake", "mydb.schema.tableB"),
builder.make_dataset_urn(
platform="snowflake", name="mydb.schema.tableA"
),
builder.make_dataset_urn_with_platform_instance(
platform="snowflake",
name="mydb.schema.tableB",
platform_instance="cloud",
),
],
downstream_urn=builder.make_dataset_urn(
"snowflake", "mydb.schema.tableC"
platform="snowflake", name="mydb.schema.tableC", env="DEV"
),
)
],