yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-23 16:48:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): presto-on-hive - Setting display name properly (#6065)

Browse Source
This commit is contained in:
Tamas Nemeth 2022-09-29 16:20:17 +02:00 committed by GitHub
parent 71f5ec7f19
commit a6930562fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 33 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py
View File

@ -462,6 +462,7 @@ class PrestoOnHiveSource(SQLAlchemySource):
properties["partitioned_columns"] = par_columns
dataset_properties = DatasetPropertiesClass(
name=key.table,
description=columns[-1]["description"],
customProperties=properties,
)
@ -627,6 +628,7 @@ class PrestoOnHiveSource(SQLAlchemySource):
"is_view": "True",
}
dataset_properties = DatasetPropertiesClass(
name=dataset.dataset_name.split(".")[-1],
description=None,
customProperties=properties,
)

23
metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json
View File

@ -176,10 +176,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test"
},
"name": "map_test",
"tags": []
}
}
@ -341,10 +342,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test"
},
"name": "union_test",
"tags": []
}
}
@ -498,10 +500,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test"
},
"name": "nested_struct_test",
"tags": []
}
}
@ -639,10 +642,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test"
},
"name": "array_struct_test",
"description": "This table has array of structs",
"tags": []
}
@ -775,10 +779,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test"
},
"name": "struct_test",
"tags": []
}
}
@ -880,10 +885,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore"
},
"name": "_test_table_underscore",
"tags": []
}
}
@ -998,11 +1004,12 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes",
"partitioned_columns": "baz"
},
"name": "pokes",
"tags": []
}
}
@ -1093,6 +1100,7 @@
"customProperties": {
"is_view": "True"
},
"name": "array_struct_test_presto_view",
"tags": []
}
},
@ -1251,6 +1259,7 @@
"customProperties": {
"is_view": "True"
},
"name": "array_struct_test_view",
"tags": []
}
},

22
metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json
View File

@ -176,10 +176,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test"
},
"name": "map_test",
"tags": []
}
}
@ -341,10 +342,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test"
},
"name": "union_test",
"tags": []
}
}
@ -498,10 +500,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test"
},
"name": "nested_struct_test",
"tags": []
}
}
@ -639,10 +642,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test"
},
"name": "array_struct_test",
"description": "This table has array of structs",
"tags": []
}
@ -775,10 +779,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test"
},
"name": "struct_test",
"tags": []
}
}
@ -880,10 +885,11 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore"
},
"name": "_test_table_underscore",
"tags": []
}
}
@ -998,11 +1004,12 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"create_date": "2022-09-20",
"create_date": "2022-09-29",
"table_type": "MANAGED_TABLE",
"table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes",
"partitioned_columns": "baz"
},
"name": "pokes",
"tags": []
}
}
@ -1106,6 +1113,7 @@
"customProperties": {
"is_view": "True"
},
"name": "array_struct_test_presto_view",
"tags": []
}
},