yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-21 07:38:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): fix issue in glue tests (#7185)

Browse Source
This commit is contained in:
Harshal Sheth 2023-01-31 00:51:21 -05:00 committed by GitHub
parent 670342ab64
commit db1a0f13f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 2306 additions and 1936 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json
View File

@ -5,8 +5,15 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "containerProperties", "aspectName": "containerProperties",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"test-database\"}, \"name\": \"test-database\", \"qualifiedName\": \"arn:aws:glue:eu-east-1:123412341234:database/test-database\"}", "json": {
"contentType": "application/json" "customProperties": {
"platform": "glue",
"instance": "PROD",
"database": "test-database"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:eu-east-1:123412341234:database/test-database"
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -19,8 +26,9 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "status", "aspectName": "status",
"aspect": { "aspect": {
"value": "{\"removed\": false}", "json": {
"contentType": "application/json" "removed": false
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -33,8 +41,9 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "dataPlatformInstance", "aspectName": "dataPlatformInstance",
"aspect": { "aspect": {
"value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", "json": {
"contentType": "application/json" "platform": "urn:li:dataPlatform:glue"
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -47,8 +56,11 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"Database\"]}", "json": {
"contentType": "application/json" "typeNames": [
"Database"
]
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -212,8 +224,11 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"table\"]}", "json": {
"contentType": "application/json" "typeNames": [
"table"
]
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -226,8 +241,9 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", "json": {
"contentType": "application/json" "container": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7"
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -392,8 +408,11 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"table\"]}", "json": {
"contentType": "application/json" "typeNames": [
"table"
]
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -406,8 +425,24 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", "json": {
"contentType": "application/json" "container": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7"
}
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "glue-2020_04_14-07_00_00"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": true
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -420,8 +455,9 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "status", "aspectName": "status",
"aspect": { "aspect": {
"value": "{\"removed\": true}", "json": {
"contentType": "application/json" "removed": true
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,

2074
metadata-ingestion/tests/unit/glue/glue_mces_golden.json
View File

File diff suppressed because it is too large Load Diff

2082
metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json
View File

File diff suppressed because it is too large Load Diff

14
metadata-ingestion/tests/unit/test_glue_source.py
View File

@ -324,14 +324,10 @@ def test_glue_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
# part of the second state # part of the second state
state1 = cast(BaseSQLAlchemyCheckpointState, checkpoint1.state) state1 = cast(BaseSQLAlchemyCheckpointState, checkpoint1.state)
state2 = cast(BaseSQLAlchemyCheckpointState, checkpoint2.state) state2 = cast(BaseSQLAlchemyCheckpointState, checkpoint2.state)
difference_urns = list( difference_urns = set(
state1.get_urns_not_in(type="*", other_checkpoint_state=state2) state1.get_urns_not_in(type="*", other_checkpoint_state=state2)
) )
assert difference_urns == {
assert len(difference_urns) == 1 "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)",
"urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba",
urn1 = ( }
"urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)"
)
assert urn1 in difference_urns