yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-24 09:08:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/mode): move sql logic to view properties aspect (#12196)

Browse Source
This commit is contained in:
Harshal Sheth 2024-12-23 16:32:49 -05:00 committed by GitHub
parent b6ea974630
commit 21ddb5538d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 55 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
metadata-ingestion/src/datahub/ingestion/source/mode.py
View File

@ -98,6 +98,7 @@ from datahub.metadata.schema_classes import (
TagPropertiesClass,
UpstreamClass,
UpstreamLineageClass,
ViewPropertiesClass,
)
from datahub.metadata.urns import QueryUrn
from datahub.sql_parsing.sqlglot_lineage import (
@ -930,16 +931,13 @@ class ModeSource(StatefulIngestionSourceBase):
dataset_props = DatasetPropertiesClass(
name=report_info.get("name") if is_mode_dataset else query_data.get("name"),
description=f"""### Source Code
``` sql
{query_data.get("raw_query")}
```
""",
description=None,
externalUrl=externalUrl,
customProperties=self.get_custom_props_from_dict(
query_data,
[
"id" "created_at",
"id",
"created_at",
"updated_at",
"last_run_id",
"data_source_id",
@ -949,7 +947,6 @@ class ModeSource(StatefulIngestionSourceBase):
],
),
)
yield (
MetadataChangeProposalWrapper(
entityUrn=query_urn,
@ -957,6 +954,16 @@ class ModeSource(StatefulIngestionSourceBase):
).as_workunit()
)
if raw_query := query_data.get("raw_query"):
yield MetadataChangeProposalWrapper(
entityUrn=query_urn,
aspect=ViewPropertiesClass(
viewLogic=raw_query,
viewLanguage=QueryLanguageClass.SQL,
materialized=False,
),
).as_workunit()
if is_mode_dataset:
space_container_key = self.gen_space_key(space_token)
yield from add_dataset_to_container(

43
metadata-ingestion/tests/integration/mode/mode_mces_golden.json
View File

@ -176,6 +176,7 @@
"datasets": [
"urn:li:dataset:(urn:li:dataPlatform:mode,5450544,PROD)"
],
"dashboards": [],
"lastModified": {
"created": {
"time": 1639169724316,
@ -253,6 +254,8 @@
"aspect": {
"json": {
"customProperties": {
"id": "19780522",
"created_at": "2024-09-02T07:38:43.755Z",
"updated_at": "2024-09-02T07:40:44.046Z",
"last_run_id": "3535709679",
"data_source_id": "44763",
@ -260,7 +263,6 @@
},
"externalUrl": "https://app.mode.com/acryl/datasets/24f66e1701b6",
"name": "Dataset 1",
"description": "### Source Code\n``` sql\n-- Returns first 100 rows from DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY\n SELECT \n\t\tAGE,\n\t\tID,\n\t\tNAME,\n\t\t_FIVETRAN_DELETED,\n\t\t_FIVETRAN_SYNCED\n FROM DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY LIMIT 100;\n\n-- Returns first 100 rows from ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER\n SELECT \n\t\tCOMMUNICATION_ACCOUNT_ID,\n\t\tID,\n\t\tMMS_CAPABLE,\n\t\tPHONE_NUMBER,\n\t\tSMS_CAPABLE,\n\t\tSTATUS,\n\t\tSTATUS_TLM,\n\t\tTLM,\n\t\tVOICE_CAPABLE,\n\t\tWHEN_CREATED\n FROM ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER LIMIT 100;\n \n \n```\n ",
"tags": []
}
},
@ -270,6 +272,24 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,5450544,PROD)",
"changeType": "UPSERT",
"aspectName": "viewProperties",
"aspect": {
"json": {
"materialized": false,
"viewLogic": "-- Returns first 100 rows from DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY\n SELECT \n\t\tAGE,\n\t\tID,\n\t\tNAME,\n\t\t_FIVETRAN_DELETED,\n\t\t_FIVETRAN_SYNCED\n FROM DATAHUB_COMMUNITY.POSTGRES_PUBLIC.COMPANY LIMIT 100;\n\n-- Returns first 100 rows from ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER\n SELECT \n\t\tCOMMUNICATION_ACCOUNT_ID,\n\t\tID,\n\t\tMMS_CAPABLE,\n\t\tPHONE_NUMBER,\n\t\tSMS_CAPABLE,\n\t\tSTATUS,\n\t\tSTATUS_TLM,\n\t\tTLM,\n\t\tVOICE_CAPABLE,\n\t\tWHEN_CREATED\n FROM ETHAN_TEST_DB.PUBLIC.ACCOUNT_PHONE_NUMBER LIMIT 100;\n \n ",
"viewLanguage": "SQL"
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "mode-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,5450544,PROD)",
@ -336,13 +356,14 @@
"aspect": {
"json": {
"customProperties": {
"id": "10149707",
"created_at": "2021-12-10T20:55:24.361Z",
"updated_at": "2021-12-10T23:12:53.273Z",
"last_run_id": "1897576958",
"data_source_id": "34499"
},
"externalUrl": "https://app.mode.com/acryl/reports/9d2da37fa91e/details/queries/6e26a9f3d4e2",
"name": "Customer and staff",
"description": "### Source Code\n``` sql\nSELECT rental.*, staff.first_name \"Staff First Name\", staff.last_name \"Staff Last Name\" FROM {{ @join_on_definition as rental }} join staff on staff.staff_id = rental.staff_id where selected_id = {{ selected_id }} \n{% form %}\nselected_id:\n type: text\n default: my_id\n{% endform %}\n```\n ",
"tags": []
}
},
@ -352,6 +373,24 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,10149707,PROD)",
"changeType": "UPSERT",
"aspectName": "viewProperties",
"aspect": {
"json": {
"materialized": false,
"viewLogic": "SELECT rental.*, staff.first_name \"Staff First Name\", staff.last_name \"Staff Last Name\" FROM {{ @join_on_definition as rental }} join staff on staff.staff_id = rental.staff_id where selected_id = {{ selected_id }} \n{% form %}\nselected_id:\n type: text\n default: my_id\n{% endform %}",
"viewLanguage": "SQL"
}
},
"systemMetadata": {
"lastObserved": 1638860400000,
"runId": "mode-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mode,10149707,PROD)",