yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-02 03:39:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: qualifiedName support + populating glue ARN (#5952)

Browse Source
This commit is contained in:
skrydal 2022-09-16 06:15:03 +02:00 committed by GitHub
parent a0f59825ec
commit a026c84691
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 44 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java
View File

@ -128,6 +128,10 @@ public class ContainerMapper {
if (gmsProperties.hasCustomProperties()) {
propertiesResult.setCustomProperties(CustomPropertiesMapper.map(gmsProperties.getCustomProperties(), entityUrn));
}
if (gmsProperties.hasQualifiedName()) {
propertiesResult.setQualifiedName(gmsProperties.getQualifiedName().toString());
}
return propertiesResult;
}

5
datahub-graphql-core/src/main/resources/entity.graphql
View File

@ -1974,6 +1974,11 @@ type ContainerProperties {
Native platform URL of the Container
"""
externalUrl: String
"""
Fully-qualified name of the Container
"""
qualifiedName: String
}
"""

2
metadata-ingestion/src/datahub/emitter/mcp_builder.py
View File

@ -159,6 +159,7 @@ def gen_containers(
owner_urn: Optional[str] = None,
external_url: Optional[str] = None,
tags: Optional[List[str]] = None,
qualified_name: Optional[str] = None,
) -> Iterable[MetadataWorkUnit]:
container_urn = make_container_urn(
guid=container_key.guid(),
@ -173,6 +174,7 @@ def gen_containers(
description=description,
customProperties=container_key.dict(exclude_none=True, by_alias=True),
externalUrl=external_url,
qualifiedName=qualified_name,
),
)
wu = MetadataWorkUnit(id=f"container-info-{name}-{container_urn}", mcp=mcp)

16
metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
View File

@ -291,6 +291,14 @@ class GlueSource(StatefulIngestionSourceBase):
run_id=self.ctx.run_id,
)
def get_glue_arn(
self, account_id: str, database: str, table: Optional[str] = None
) -> str:
prefix = f"arn:aws:glue:{self.source_config.aws_region}:{account_id}"
if table:
return f"{prefix}:table/{database}/{table}"
return f"{prefix}:database/{database}"
@classmethod
def create(cls, config_dict, ctx):
config = GlueSourceConfig.parse_obj(config_dict)
@ -925,6 +933,9 @@ class GlueSource(StatefulIngestionSourceBase):
sub_types=["Database"],
domain_urn=domain_urn,
description=database.get("Description"),
qualified_name=self.get_glue_arn(
account_id=database["CatalogId"], database=database["Name"]
),
)
for wu in container_workunits:
@ -1124,6 +1135,11 @@ class GlueSource(StatefulIngestionSourceBase):
},
uri=table.get("Location"),
tags=[],
qualifiedName=self.get_glue_arn(
account_id=table["CatalogId"],
database=table["DatabaseName"],
table=table["Name"],
),
)
def get_s3_tags() -> Optional[GlobalTagsClass]:

4
metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json
View File

@ -5,7 +5,7 @@
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"test-database\"}, \"name\": \"test-database\"}",
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"test-database\"}, \"name\": \"test-database\", \"qualifiedName\": \"arn:aws:glue:eu-east-1:123412341234:database/test-database\"}",
"contentType": "application/json"
},
"systemMetadata": {
@ -74,6 +74,7 @@
"SortColumns": "[]",
"StoredAsSubDirectories": "False"
},
"qualifiedName": "arn:aws:glue:eu-east-1:795586375822:table/test-database/test_jsons_markers",
"tags": []
}
},
@ -252,6 +253,7 @@
"SortColumns": "[]",
"StoredAsSubDirectories": "False"
},
"qualifiedName": "arn:aws:glue:eu-east-1:795586375822:table/test-database/test_parquet",
"tags": []
}
},

7
metadata-ingestion/tests/unit/glue/glue_mces_golden.json
View File

@ -5,7 +5,7 @@
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"flights-database\"}, \"name\": \"flights-database\"}",
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"flights-database\"}, \"name\": \"flights-database\", \"qualifiedName\": \"arn:aws:glue:us-west-2:123412341234:database/flights-database\"}",
"contentType": "application/json"
}
},
@ -63,6 +63,7 @@
"SortColumns": "[]",
"StoredAsSubDirectories": "False"
},
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:table/flights-database/avro",
"tags": []
}
},
@ -254,7 +255,7 @@
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"test-database\"}, \"name\": \"test-database\"}",
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"test-database\"}, \"name\": \"test-database\", \"qualifiedName\": \"arn:aws:glue:us-west-2:123412341234:database/test-database\"}",
"contentType": "application/json"
}
},
@ -311,6 +312,7 @@
"SortColumns": "[]",
"StoredAsSubDirectories": "False"
},
"qualifiedName": "arn:aws:glue:us-west-2:795586375822:table/test-database/test_jsons_markers",
"tags": []
}
},
@ -489,6 +491,7 @@
"SortColumns": "[]",
"StoredAsSubDirectories": "False"
},
"qualifiedName": "arn:aws:glue:us-west-2:795586375822:table/test-database/test_parquet",
"tags": []
}
},

7
metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json
View File

@ -5,7 +5,7 @@
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"some_instance_name\", \"database\": \"flights-database\"}, \"name\": \"flights-database\"}",
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"some_instance_name\", \"database\": \"flights-database\"}, \"name\": \"flights-database\", \"qualifiedName\": \"arn:aws:glue:us-west-2:123412341234:database/flights-database\"}",
"contentType": "application/json"
}
},
@ -63,6 +63,7 @@
"SortColumns": "[]",
"StoredAsSubDirectories": "False"
},
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:table/flights-database/avro",
"tags": []
}
},
@ -255,7 +256,7 @@
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"some_instance_name\", \"database\": \"test-database\"}, \"name\": \"test-database\"}",
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"some_instance_name\", \"database\": \"test-database\"}, \"name\": \"test-database\", \"qualifiedName\": \"arn:aws:glue:us-west-2:123412341234:database/test-database\"}",
"contentType": "application/json"
}
},
@ -312,6 +313,7 @@
"SortColumns": "[]",
"StoredAsSubDirectories": "False"
},
"qualifiedName": "arn:aws:glue:us-west-2:795586375822:table/test-database/test_jsons_markers",
"tags": []
}
},
@ -491,6 +493,7 @@
"SortColumns": "[]",
"StoredAsSubDirectories": "False"
},
"qualifiedName": "arn:aws:glue:us-west-2:795586375822:table/test-database/test_parquet",
"tags": []
}
},

6
metadata-ingestion/tests/unit/test_glue_source_stubs.py
View File

@ -34,8 +34,10 @@ get_databases_response = {
},
]
}
databases_1 = {"flights-database": {"Name": "flights-database"}}
databases_2 = {"test-database": {"Name": "test-database"}}
databases_1 = {
"flights-database": {"Name": "flights-database", "CatalogId": "123412341234"}
}
databases_2 = {"test-database": {"Name": "test-database", "CatalogId": "123412341234"}}
tables_1 = [
{
"Name": "avro",