yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-06 04:26:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add queries in sample data for lineage (#8674)

Browse Source
This commit is contained in:
Mayur Singal 2022-11-14 13:11:16 +05:30 committed by GitHub
parent 01bc9f1cfe
commit 18cd1c99ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ingestion/examples/sample_data/lineage/lineage.json
View File

@ -8,6 +8,7 @@
"fqn": "sample_data.ecommerce_db.shopify.dim_address",
"type": "table"
},
"sql_query": "select * from ecommerce_db.shopify.raw_customer",
"edge_meta": { "fqn": "sample_airflow.dim_address_etl", "type": "pipeline" }
},
{
@ -19,6 +20,7 @@
"fqn": "sample_data.ecommerce_db.shopify.\"dim.product\"",
"type": "table"
},
"sql_query": "select * from sample_data.ecommerce_db.shopify.raw_order",
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" }
},
{
@ -30,7 +32,8 @@
"fqn": "sample_data.ecommerce_db.shopify.\"dim.product.variant\"",
"type": "table"
},
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" }
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" },
"sql_query": "select * from sample_data.ecommerce_db.shopify.raw_order"
},
{
"from": {
@ -41,7 +44,8 @@
"fqn": "sample_data.ecommerce_db.shopify.\"dim.product\"",
"type": "table"
},
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" }
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" },
"sql_query": "insert into sample_data.ecommerce_db.shopify.\"dim.product\" as select * from sample_data.ecommerce_db.shopify.raw_customer"
},
{
"from": {
@ -52,6 +56,8 @@
"fqn": "sample_data.ecommerce_db.shopify.\"dim.product.variant\"",
"type": "table"
},
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" }
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" },
"sql_query": "create ecommerce_db.shopify.\"dim.product.variant\" as select * from sample_data.ecommerce_db.shopify.raw_customer"
}
]

4
ingestion/src/metadata/ingestion/source/database/sample_data.py
View File

@ -728,7 +728,9 @@ class SampleDataSource(
edge_entity_ref = get_lineage_entity_ref(
edge["edge_meta"], self.metadata_config
)
lineage_details = LineageDetails(pipeline=edge_entity_ref)
lineage_details = LineageDetails(
pipeline=edge_entity_ref, sqlQuery=edge.get("sql_query")
)
lineage = AddLineageRequest(
edge=EntitiesEdge(
fromEntity=from_entity_ref,