diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py index 1bad0e0a1b..874dc72ac2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py @@ -462,6 +462,7 @@ class PrestoOnHiveSource(SQLAlchemySource): properties["partitioned_columns"] = par_columns dataset_properties = DatasetPropertiesClass( + name=key.table, description=columns[-1]["description"], customProperties=properties, ) @@ -627,6 +628,7 @@ class PrestoOnHiveSource(SQLAlchemySource): "is_view": "True", } dataset_properties = DatasetPropertiesClass( + name=dataset.dataset_name.split(".")[-1], description=None, customProperties=properties, ) diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json index 1b0c2609e5..f0d21ea8d2 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json @@ -176,10 +176,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" }, + "name": "map_test", "tags": [] } } @@ -341,10 +342,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" }, + "name": "union_test", "tags": [] } } @@ -498,10 +500,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" }, + "name": "nested_struct_test", "tags": [] } } @@ -639,10 +642,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" }, + "name": "array_struct_test", "description": "This table has array of structs", "tags": [] } @@ -775,10 +779,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" }, + "name": "struct_test", "tags": [] } } @@ -880,10 +885,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" }, + "name": "_test_table_underscore", "tags": [] } } @@ -998,11 +1004,12 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", "partitioned_columns": "baz" }, + "name": "pokes", "tags": [] } } @@ -1093,6 +1100,7 @@ "customProperties": { "is_view": "True" }, + "name": "array_struct_test_presto_view", "tags": [] } }, @@ -1251,6 +1259,7 @@ "customProperties": { "is_view": "True" }, + "name": "array_struct_test_view", "tags": [] } }, diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json index c2403ae18b..5125aa2c19 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json @@ -176,10 +176,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" }, + "name": "map_test", "tags": [] } } @@ -341,10 +342,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" }, + "name": "union_test", "tags": [] } } @@ -498,10 +500,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" }, + "name": "nested_struct_test", "tags": [] } } @@ -639,10 +642,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" }, + "name": "array_struct_test", "description": "This table has array of structs", "tags": [] } @@ -775,10 +779,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" }, + "name": "struct_test", "tags": [] } } @@ -880,10 +885,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" }, + "name": "_test_table_underscore", "tags": [] } } @@ -998,11 +1004,12 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-29", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", "partitioned_columns": "baz" }, + "name": "pokes", "tags": [] } } @@ -1106,6 +1113,7 @@ "customProperties": { "is_view": "True" }, + "name": "array_struct_test_presto_view", "tags": [] } },