yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-23 00:28:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingestion/glue): ensure date formatting works on all platforms for aws glue (#10836)

Browse Source
This commit is contained in:
sagar-salvi-apptware 2024-07-03 18:05:37 +05:30 committed by GitHub
parent e45f7a4167
commit b8af2b9d69
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 115 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
View File

@ -901,7 +901,7 @@ class GlueSource(StatefulIngestionSourceBase):
parameters["LocationUri"] = database["LocationUri"] parameters["LocationUri"] = database["LocationUri"]
if database.get("CreateTime") is not None: if database.get("CreateTime") is not None:
create_time: datetime.datetime = database["CreateTime"] create_time: datetime.datetime = database["CreateTime"]
parameters["CreateTime"] = create_time.strftime("%B %-d, %Y at %H:%M:%S") parameters["CreateTime"] = create_time.strftime("%B %d, %Y at %H:%M:%S")
yield from gen_containers( yield from gen_containers(
container_key=database_container_key, container_key=database_container_key,
name=database["Name"], name=database["Name"],

2
metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json
View File

@ -11,7 +11,7 @@
"instance": "delta_platform_instance", "instance": "delta_platform_instance",
"env": "PROD", "env": "PROD",
"database": "delta-database", "database": "delta-database",
"CreateTime": "June 9, 2021 at 14:14:19" "CreateTime": "June 09, 2021 at 14:14:19"
}, },
"name": "delta-database", "name": "delta-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database" "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database"

2
metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json
View File

@ -11,7 +11,7 @@
"instance": "delta_platform_instance", "instance": "delta_platform_instance",
"env": "PROD", "env": "PROD",
"database": "delta-database", "database": "delta-database",
"CreateTime": "June 9, 2021 at 14:14:19" "CreateTime": "June 09, 2021 at 14:14:19"
}, },
"name": "delta-database", "name": "delta-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database" "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database"

110
metadata-ingestion/tests/unit/glue/glue_mces_golden.json
View File

@ -13,7 +13,7 @@
"param1": "value1", "param1": "value1",
"param2": "value2", "param2": "value2",
"LocationUri": "s3://test-bucket/test-prefix", "LocationUri": "s3://test-bucket/test-prefix",
"CreateTime": "June 9, 2021 at 14:14:19" "CreateTime": "June 09, 2021 at 14:14:19"
}, },
"name": "flights-database", "name": "flights-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database" "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database"
@ -55,6 +55,59 @@
} }
} }
}, },
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "glue",
"env": "PROD",
"database": "test-database",
"CreateTime": "June 01, 2021 at 14:55:02"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:glue"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Database"
]
}
}
},
{ {
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7", "entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7",
@ -66,7 +119,7 @@
"platform": "glue", "platform": "glue",
"env": "PROD", "env": "PROD",
"database": "empty-database", "database": "empty-database",
"CreateTime": "June 1, 2021 at 14:55:13" "CreateTime": "June 01, 2021 at 14:55:13"
}, },
"name": "empty-database", "name": "empty-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database" "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database"
@ -336,59 +389,6 @@
} }
} }
}, },
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "glue",
"env": "PROD",
"database": "test-database",
"CreateTime": "June 1, 2021 at 14:55:02"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:glue"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Database"
]
}
}
},
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {

114
metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json
View File

@ -14,7 +14,7 @@
"param1": "value1", "param1": "value1",
"param2": "value2", "param2": "value2",
"LocationUri": "s3://test-bucket/test-prefix", "LocationUri": "s3://test-bucket/test-prefix",
"CreateTime": "June 9, 2021 at 14:14:19" "CreateTime": "June 09, 2021 at 14:14:19"
}, },
"name": "flights-database", "name": "flights-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database" "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database"
@ -57,6 +57,61 @@
} }
} }
}, },
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "glue",
"instance": "some_instance_name",
"env": "PROD",
"database": "test-database",
"CreateTime": "June 01, 2021 at 14:55:02"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:glue",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Database"
]
}
}
},
{ {
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4", "entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4",
@ -69,7 +124,7 @@
"instance": "some_instance_name", "instance": "some_instance_name",
"env": "PROD", "env": "PROD",
"database": "empty-database", "database": "empty-database",
"CreateTime": "June 1, 2021 at 14:55:13" "CreateTime": "June 01, 2021 at 14:55:13"
}, },
"name": "empty-database", "name": "empty-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database" "qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database"
@ -341,61 +396,6 @@
} }
} }
}, },
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "glue",
"instance": "some_instance_name",
"env": "PROD",
"database": "test-database",
"CreateTime": "June 1, 2021 at 14:55:02"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:glue",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Database"
]
}
}
},
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {