yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-03 12:16:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/glue): database parameters extraction (#10665)

Browse Source
This commit is contained in:
skrydal 2024-06-11 20:50:46 +02:00 committed by GitHub
parent 46dbb10940
commit b9e71a61b1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 31 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
View File

@ -1,3 +1,4 @@
import datetime
import json
import logging
from collections import defaultdict
@ -895,6 +896,12 @@ class GlueSource(StatefulIngestionSourceBase):
) -> Iterable[MetadataWorkUnit]:
domain_urn = self._gen_domain_urn(database["Name"])
database_container_key = self.gen_database_key(database["Name"])
parameters = database.get("Parameters", {})
if database.get("LocationUri") is not None:
parameters["LocationUri"] = database["LocationUri"]
if database.get("CreateTime") is not None:
create_time: datetime.datetime = database["CreateTime"]
parameters["CreateTime"] = create_time.strftime("%B %-d, %Y at %H:%M:%S")
yield from gen_containers(
container_key=database_container_key,
name=database["Name"],
@ -904,6 +911,7 @@ class GlueSource(StatefulIngestionSourceBase):
qualified_name=self.get_glue_arn(
account_id=database["CatalogId"], database=database["Name"]
),
extra_properties=parameters,
)
def add_table_to_database_container(

3
metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json
View File

@ -10,7 +10,8 @@
"platform": "glue",
"instance": "delta_platform_instance",
"env": "PROD",
"database": "delta-database"
"database": "delta-database",
"CreateTime": "June 9, 2021 at 14:14:19"
},
"name": "delta-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database"

3
metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json
View File

@ -10,7 +10,8 @@
"platform": "glue",
"instance": "delta_platform_instance",
"env": "PROD",
"database": "delta-database"
"database": "delta-database",
"CreateTime": "June 9, 2021 at 14:14:19"
},
"name": "delta-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database"

9
metadata-ingestion/tests/unit/glue/glue_mces_golden.json
View File

@ -9,7 +9,11 @@
"customProperties": {
"platform": "glue",
"env": "PROD",
"database": "flights-database"
"database": "flights-database",
"param1": "value1",
"param2": "value2",
"LocationUri": "s3://test-bucket/test-prefix",
"CreateTime": "June 9, 2021 at 14:14:19"
},
"name": "flights-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database"
@ -288,7 +292,8 @@
"customProperties": {
"platform": "glue",
"env": "PROD",
"database": "test-database"
"database": "test-database",
"CreateTime": "June 1, 2021 at 14:55:02"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"

12
metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json
View File

@ -10,7 +10,11 @@
"platform": "glue",
"instance": "some_instance_name",
"env": "PROD",
"database": "flights-database"
"database": "flights-database",
"param1": "value1",
"param2": "value2",
"LocationUri": "s3://test-bucket/test-prefix",
"CreateTime": "June 9, 2021 at 14:14:19"
},
"name": "flights-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database"
@ -235,6 +239,7 @@
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -292,7 +297,8 @@
"platform": "glue",
"instance": "some_instance_name",
"env": "PROD",
"database": "test-database"
"database": "test-database",
"CreateTime": "June 1, 2021 at 14:55:02"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
@ -474,6 +480,7 @@
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
@ -660,6 +667,7 @@
"type": "DATAOWNER"
}
],
"ownerTypes": {},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"

2
metadata-ingestion/tests/unit/test_glue_source_stubs.py
View File

@ -61,6 +61,8 @@ get_databases_response = {
}
],
"CatalogId": "123412341234",
"LocationUri": "s3://test-bucket/test-prefix",
"Parameters": {"param1": "value1", "param2": "value2"},
},
{
"Name": "test-database",