diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index fdacec76b3..7dd51d5b20 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -1029,9 +1029,7 @@ class SnowflakeV2Source( entityUrn=dataset_urn, aspect=schema_metadata ).as_workunit() - dataset_properties = self.get_dataset_properties( - table, schema_name, db_name, dataset_name - ) + dataset_properties = self.get_dataset_properties(table, schema_name, db_name) yield MetadataChangeProposalWrapper( entityUrn=dataset_urn, aspect=dataset_properties @@ -1098,7 +1096,12 @@ class SnowflakeV2Source( entityUrn=dataset_urn, aspect=view_properties_aspect ).as_workunit() - def get_dataset_properties(self, table, schema_name, db_name, dataset_name): + def get_dataset_properties( + self, + table: Union[SnowflakeTable, SnowflakeView], + schema_name: str, + db_name: str, + ) -> DatasetProperties: return DatasetProperties( name=table.name, created=TimeStamp(time=int(table.created.timestamp() * 1000)) @@ -1110,7 +1113,7 @@ class SnowflakeV2Source( if table.created is not None else None, description=table.comment, - qualifiedName=dataset_name, + qualifiedName=f"{db_name}.{schema_name}.{table.name}", customProperties={}, externalUrl=self.get_external_url_for_table( table.name, diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index dd56eff79c..a424b258e6 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -475,20 +475,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/", "name": "TABLE_1", - "qualifiedName": "test_db.test_schema.table_1", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1", "description": "Comment for Table", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { @@ -768,20 +768,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/", "name": "TABLE_2", - "qualifiedName": "test_db.test_schema.table_2", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2", "description": "Comment for Table", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { @@ -1061,20 +1061,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/", "name": "TABLE_3", - "qualifiedName": "test_db.test_schema.table_3", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3", "description": "Comment for Table", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { @@ -1354,20 +1354,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/", "name": "TABLE_4", - "qualifiedName": "test_db.test_schema.table_4", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4", "description": "Comment for Table", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { @@ -1647,20 +1647,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/", "name": "TABLE_5", - "qualifiedName": "test_db.test_schema.table_5", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5", "description": "Comment for Table", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { @@ -1940,20 +1940,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/", "name": "TABLE_6", - "qualifiedName": "test_db.test_schema.table_6", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6", "description": "Comment for Table", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { @@ -2233,20 +2233,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/", "name": "TABLE_7", - "qualifiedName": "test_db.test_schema.table_7", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7", "description": "Comment for Table", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { @@ -2526,20 +2526,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/", "name": "TABLE_8", - "qualifiedName": "test_db.test_schema.table_8", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8", "description": "Comment for Table", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { @@ -2819,20 +2819,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/", "name": "TABLE_9", - "qualifiedName": "test_db.test_schema.table_9", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9", "description": "Comment for Table", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { @@ -3112,20 +3112,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/", "name": "TABLE_10", - "qualifiedName": "test_db.test_schema.table_10", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10", "description": "Comment for Table", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { @@ -3395,20 +3395,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_1/", "name": "VIEW_1", - "qualifiedName": "test_db.test_schema.view_1", + "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1", "description": "Comment for View", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { @@ -3703,20 +3703,20 @@ "customProperties": {}, "externalUrl": "https://app.snowflake.com/ap-south-1/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/VIEW_2/", "name": "VIEW_2", - "qualifiedName": "test_db.test_schema.view_2", + "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_2", "description": "Comment for View", "created": { - "time": 1623110400000 + "time": 1623090600000 }, "lastModified": { - "time": 1623110400000 + "time": 1623090600000 }, "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00" + "lastObserved": 1615443388097, + "runId": "snowflake-2023_08_04-09_52_28" } }, { diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json index 21c2c656df..7687b99ac8 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json @@ -388,7 +388,7 @@ "customProperties": {}, "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/", "name": "TABLE_1", - "qualifiedName": "test_db.test_schema.table_1", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1", "description": "Comment for Table", "created": { "time": 1623110400000 @@ -648,7 +648,7 @@ "customProperties": {}, "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/", "name": "TABLE_2", - "qualifiedName": "test_db.test_schema.table_2", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2", "description": "Comment for Table", "created": { "time": 1623110400000 @@ -908,7 +908,7 @@ "customProperties": {}, "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/", "name": "TABLE_3", - "qualifiedName": "test_db.test_schema.table_3", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3", "description": "Comment for Table", "created": { "time": 1623110400000 @@ -1168,7 +1168,7 @@ "customProperties": {}, "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/", "name": "TABLE_4", - "qualifiedName": "test_db.test_schema.table_4", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4", "description": "Comment for Table", "created": { "time": 1623110400000 @@ -1428,7 +1428,7 @@ "customProperties": {}, "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/", "name": "TABLE_5", - "qualifiedName": "test_db.test_schema.table_5", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5", "description": "Comment for Table", "created": { "time": 1623110400000 @@ -1688,7 +1688,7 @@ "customProperties": {}, "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/", "name": "TABLE_6", - "qualifiedName": "test_db.test_schema.table_6", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6", "description": "Comment for Table", "created": { "time": 1623110400000 @@ -1948,7 +1948,7 @@ "customProperties": {}, "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/", "name": "TABLE_7", - "qualifiedName": "test_db.test_schema.table_7", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7", "description": "Comment for Table", "created": { "time": 1623110400000 @@ -2208,7 +2208,7 @@ "customProperties": {}, "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/", "name": "TABLE_8", - "qualifiedName": "test_db.test_schema.table_8", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8", "description": "Comment for Table", "created": { "time": 1623110400000 @@ -2468,7 +2468,7 @@ "customProperties": {}, "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/", "name": "TABLE_9", - "qualifiedName": "test_db.test_schema.table_9", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9", "description": "Comment for Table", "created": { "time": 1623110400000 @@ -2728,7 +2728,7 @@ "customProperties": {}, "externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/", "name": "TABLE_10", - "qualifiedName": "test_db.test_schema.table_10", + "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10", "description": "Comment for Table", "created": { "time": 1623110400000