yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-30 03:18:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/snowflake): maintain qualified name casing, do not lowercase (#8574)

Browse Source
This commit is contained in:
Mayuri Nehate 2023-08-04 23:13:22 +05:30 committed by GitHub
parent ef15861d0d
commit b4e104f190
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 78 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py
View File

@ -1029,9 +1029,7 @@ class SnowflakeV2Source(
entityUrn=dataset_urn, aspect=schema_metadata
).as_workunit()
dataset_properties = self.get_dataset_properties(
table, schema_name, db_name, dataset_name
)
dataset_properties = self.get_dataset_properties(table, schema_name, db_name)
yield MetadataChangeProposalWrapper(
entityUrn=dataset_urn, aspect=dataset_properties
@ -1098,7 +1096,12 @@ class SnowflakeV2Source(
entityUrn=dataset_urn, aspect=view_properties_aspect
).as_workunit()
def get_dataset_properties(self, table, schema_name, db_name, dataset_name):
def get_dataset_properties(
self,
table: Union[SnowflakeTable, SnowflakeView],
schema_name: str,
db_name: str,
) -> DatasetProperties:
return DatasetProperties(
name=table.name,
created=TimeStamp(time=int(table.created.timestamp() * 1000))
@ -1110,7 +1113,7 @@ class SnowflakeV2Source(
if table.created is not None
else None,
description=table.comment,
qualifiedName=dataset_name,
qualifiedName=f"{db_name}.{schema_name}.{table.name}",
customProperties={},
externalUrl=self.get_external_url_for_table(
table.name,

120
metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
View File

@ -475,20 +475,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/",
"name": "TABLE_1",
"qualifiedName": "test_db.test_schema.table_1",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
"description": "Comment for Table",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{
@ -768,20 +768,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/",
"name": "TABLE_2",
"qualifiedName": "test_db.test_schema.table_2",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
"description": "Comment for Table",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{
@ -1061,20 +1061,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/",
"name": "TABLE_3",
"qualifiedName": "test_db.test_schema.table_3",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
"description": "Comment for Table",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{
@ -1354,20 +1354,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/",
"name": "TABLE_4",
"qualifiedName": "test_db.test_schema.table_4",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
"description": "Comment for Table",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{
@ -1647,20 +1647,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/",
"name": "TABLE_5",
"qualifiedName": "test_db.test_schema.table_5",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5",
"description": "Comment for Table",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{
@ -1940,20 +1940,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/",
"name": "TABLE_6",
"qualifiedName": "test_db.test_schema.table_6",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6",
"description": "Comment for Table",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{
@ -2233,20 +2233,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/",
"name": "TABLE_7",
"qualifiedName": "test_db.test_schema.table_7",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7",
"description": "Comment for Table",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{
@ -2526,20 +2526,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/",
"name": "TABLE_8",
"qualifiedName": "test_db.test_schema.table_8",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8",
"description": "Comment for Table",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{
@ -2819,20 +2819,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/",
"name": "TABLE_9",
"qualifiedName": "test_db.test_schema.table_9",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9",
"description": "Comment for Table",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{
@ -3112,20 +3112,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/",
"name": "TABLE_10",
"qualifiedName": "test_db.test_schema.table_10",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10",
"description": "Comment for Table",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{
@ -3395,20 +3395,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/",
"name": "VIEW_1",
"qualifiedName": "test_db.test_schema.view_1",
"qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1",
"description": "Comment for View",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{
@ -3703,20 +3703,20 @@
"customProperties": {},
"externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/",
"name": "VIEW_2",
"qualifiedName": "test_db.test_schema.view_2",
"qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_2",
"description": "Comment for View",
"created": {
"time": 1623110400000
"time": 1623090600000
},
"lastModified": {
"time": 1623110400000
"time": 1623090600000
},
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00"
"lastObserved": 1615443388097,
"runId": "snowflake-2023_08_04-09_52_28"
}
},
{

20
metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
View File

@ -388,7 +388,7 @@
"customProperties": {},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/",
"name": "TABLE_1",
"qualifiedName": "test_db.test_schema.table_1",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
"description": "Comment for Table",
"created": {
"time": 1623110400000
@ -648,7 +648,7 @@
"customProperties": {},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/",
"name": "TABLE_2",
"qualifiedName": "test_db.test_schema.table_2",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
"description": "Comment for Table",
"created": {
"time": 1623110400000
@ -908,7 +908,7 @@
"customProperties": {},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/",
"name": "TABLE_3",
"qualifiedName": "test_db.test_schema.table_3",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
"description": "Comment for Table",
"created": {
"time": 1623110400000
@ -1168,7 +1168,7 @@
"customProperties": {},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/",
"name": "TABLE_4",
"qualifiedName": "test_db.test_schema.table_4",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
"description": "Comment for Table",
"created": {
"time": 1623110400000
@ -1428,7 +1428,7 @@
"customProperties": {},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/",
"name": "TABLE_5",
"qualifiedName": "test_db.test_schema.table_5",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5",
"description": "Comment for Table",
"created": {
"time": 1623110400000
@ -1688,7 +1688,7 @@
"customProperties": {},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/",
"name": "TABLE_6",
"qualifiedName": "test_db.test_schema.table_6",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6",
"description": "Comment for Table",
"created": {
"time": 1623110400000
@ -1948,7 +1948,7 @@
"customProperties": {},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/",
"name": "TABLE_7",
"qualifiedName": "test_db.test_schema.table_7",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7",
"description": "Comment for Table",
"created": {
"time": 1623110400000
@ -2208,7 +2208,7 @@
"customProperties": {},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/",
"name": "TABLE_8",
"qualifiedName": "test_db.test_schema.table_8",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8",
"description": "Comment for Table",
"created": {
"time": 1623110400000
@ -2468,7 +2468,7 @@
"customProperties": {},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/",
"name": "TABLE_9",
"qualifiedName": "test_db.test_schema.table_9",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9",
"description": "Comment for Table",
"created": {
"time": 1623110400000
@ -2728,7 +2728,7 @@
"customProperties": {},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/",
"name": "TABLE_10",
"qualifiedName": "test_db.test_schema.table_10",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10",
"description": "Comment for Table",
"created": {
"time": 1623110400000