yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-21 07:38:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): fix issue in glue tests (#7185)

Browse Source
This commit is contained in:
Harshal Sheth 2023-01-31 00:51:21 -05:00 committed by GitHub
parent 670342ab64
commit db1a0f13f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 2306 additions and 1936 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
metadata-ingestion/tests/unit/glue/glue_deleted_actor_mces_golden.json
View File

@ -5,8 +5,15 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "containerProperties", "aspectName": "containerProperties",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"test-database\"}, \"name\": \"test-database\", \"qualifiedName\": \"arn:aws:glue:eu-east-1:123412341234:database/test-database\"}", "json": {
"contentType": "application/json" "customProperties": {
"platform": "glue",
"instance": "PROD",
"database": "test-database"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:eu-east-1:123412341234:database/test-database"
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -19,8 +26,9 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "status", "aspectName": "status",
"aspect": { "aspect": {
"value": "{\"removed\": false}", "json": {
"contentType": "application/json" "removed": false
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -33,8 +41,9 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "dataPlatformInstance", "aspectName": "dataPlatformInstance",
"aspect": { "aspect": {
"value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", "json": {
"contentType": "application/json" "platform": "urn:li:dataPlatform:glue"
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -47,8 +56,11 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"Database\"]}", "json": {
"contentType": "application/json" "typeNames": [
"Database"
]
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -212,8 +224,11 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"table\"]}", "json": {
"contentType": "application/json" "typeNames": [
"table"
]
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -226,8 +241,9 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", "json": {
"contentType": "application/json" "container": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7"
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -392,8 +408,11 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"table\"]}", "json": {
"contentType": "application/json" "typeNames": [
"table"
]
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -406,8 +425,24 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", "json": {
"contentType": "application/json" "container": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7"
}
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "glue-2020_04_14-07_00_00"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": true
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,
@ -420,8 +455,9 @@
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "status", "aspectName": "status",
"aspect": { "aspect": {
"value": "{\"removed\": true}", "json": {
"contentType": "application/json" "removed": true
}
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1586847600000, "lastObserved": 1586847600000,

346
metadata-ingestion/tests/unit/glue/glue_mces_golden.json
View File

@ -1,45 +1,57 @@
[ [
{ {
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "containerProperties", "aspectName": "containerProperties",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"flights-database\"}, \"name\": \"flights-database\", \"qualifiedName\": \"arn:aws:glue:us-west-2:123412341234:database/flights-database\"}", "json": {
"contentType": "application/json" "customProperties": {
} "platform": "glue",
"instance": "PROD",
"database": "flights-database"
}, },
{ "name": "flights-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database"
}
}
},
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "status", "aspectName": "status",
"aspect": { "aspect": {
"value": "{\"removed\": false}", "json": {
"contentType": "application/json" "removed": false
} }
}, }
{ },
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "dataPlatformInstance", "aspectName": "dataPlatformInstance",
"aspect": { "aspect": {
"value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", "json": {
"contentType": "application/json" "platform": "urn:li:dataPlatform:glue"
} }
}, }
{ },
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"Database\"]}", "json": {
"contentType": "application/json" "typeNames": [
"Database"
]
} }
}, }
{ },
{
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)",
@ -238,68 +250,84 @@
] ]
} }
} }
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"table\"]}", "json": {
"contentType": "application/json" "typeNames": [
"table"
]
} }
}, }
{ },
{
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba\"}", "json": {
"contentType": "application/json" "container": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba"
} }
}, }
{ },
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "containerProperties", "aspectName": "containerProperties",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"test-database\"}, \"name\": \"test-database\", \"qualifiedName\": \"arn:aws:glue:us-west-2:123412341234:database/test-database\"}", "json": {
"contentType": "application/json" "customProperties": {
} "platform": "glue",
"instance": "PROD",
"database": "test-database"
}, },
{ "name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
}
}
},
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "status", "aspectName": "status",
"aspect": { "aspect": {
"value": "{\"removed\": false}", "json": {
"contentType": "application/json" "removed": false
} }
}, }
{ },
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "dataPlatformInstance", "aspectName": "dataPlatformInstance",
"aspect": { "aspect": {
"value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", "json": {
"contentType": "application/json" "platform": "urn:li:dataPlatform:glue"
} }
}, }
{ },
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"Database\"]}", "json": {
"contentType": "application/json" "typeNames": [
"Database"
]
} }
}, }
{ },
{
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)",
@ -457,28 +485,32 @@
] ]
} }
} }
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"table\"]}", "json": {
"contentType": "application/json" "typeNames": [
"table"
]
} }
}, }
{ },
{
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_jsons_markers,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", "json": {
"contentType": "application/json" "container": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7"
} }
}, }
{ },
{
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)",
@ -637,28 +669,32 @@
] ]
} }
} }
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"table\"]}", "json": {
"contentType": "application/json" "typeNames": [
"table"
]
} }
}, }
{ },
{
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", "json": {
"contentType": "application/json" "container": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7"
} }
}, }
{ },
{
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": {
"urn": "urn:li:dataFlow:(glue,test-job-1,PROD)", "urn": "urn:li:dataFlow:(glue,test-job-1,PROD)",
@ -679,8 +715,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": {
"urn": "urn:li:dataFlow:(glue,test-job-2,PROD)", "urn": "urn:li:dataFlow:(glue,test-job-2,PROD)",
@ -701,8 +737,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)",
@ -734,8 +770,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform1_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform1_job1)",
@ -767,8 +803,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)",
@ -800,8 +836,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Join-Transform3_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Join-Transform3_job1)",
@ -834,8 +870,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)",
@ -865,8 +901,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform5_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform5_job1)",
@ -896,8 +932,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)",
@ -921,8 +957,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SplitFields-Transform0_job2)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SplitFields-Transform0_job2)",
@ -956,8 +992,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)",
@ -989,8 +1025,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)",
@ -1022,8 +1058,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SelectFields-Transform3_job2)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SelectFields-Transform3_job2)",
@ -1057,8 +1093,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)",
@ -1082,5 +1118,137 @@
] ]
} }
} }
},
{
"entityType": "dataFlow",
"entityUrn": "urn:li:dataFlow:(glue,test-job-1,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
} }
}
},
{
"entityType": "dataFlow",
"entityUrn": "urn:li:dataFlow:(glue,test-job-2,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform1_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform5_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Join-Transform3_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SelectFields-Transform3_job2)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SplitFields-Transform0_job2)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
}
] ]

348
metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json
View File

@ -1,45 +1,58 @@
[ [
{ {
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:7d53111f2c71396ea6f6d26c84770665", "entityUrn": "urn:li:container:7d53111f2c71396ea6f6d26c84770665",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "containerProperties", "aspectName": "containerProperties",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"some_instance_name\", \"database\": \"flights-database\"}, \"name\": \"flights-database\", \"qualifiedName\": \"arn:aws:glue:us-west-2:123412341234:database/flights-database\"}", "json": {
"contentType": "application/json" "customProperties": {
} "platform": "glue",
"instance": "some_instance_name",
"database": "flights-database"
}, },
{ "name": "flights-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database"
}
}
},
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:7d53111f2c71396ea6f6d26c84770665", "entityUrn": "urn:li:container:7d53111f2c71396ea6f6d26c84770665",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "status", "aspectName": "status",
"aspect": { "aspect": {
"value": "{\"removed\": false}", "json": {
"contentType": "application/json" "removed": false
} }
}, }
{ },
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:7d53111f2c71396ea6f6d26c84770665", "entityUrn": "urn:li:container:7d53111f2c71396ea6f6d26c84770665",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "dataPlatformInstance", "aspectName": "dataPlatformInstance",
"aspect": { "aspect": {
"value": "{\"platform\": \"urn:li:dataPlatform:glue\", \"instance\": \"urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)\"}", "json": {
"contentType": "application/json" "platform": "urn:li:dataPlatform:glue",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)"
} }
}, }
{ },
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:7d53111f2c71396ea6f6d26c84770665", "entityUrn": "urn:li:container:7d53111f2c71396ea6f6d26c84770665",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"Database\"]}", "json": {
"contentType": "application/json" "typeNames": [
"Database"
]
} }
}, }
{ },
{
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)",
@ -239,68 +252,85 @@
] ]
} }
} }
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"table\"]}", "json": {
"contentType": "application/json" "typeNames": [
"table"
]
} }
}, }
{ },
{
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:7d53111f2c71396ea6f6d26c84770665\"}", "json": {
"contentType": "application/json" "container": "urn:li:container:7d53111f2c71396ea6f6d26c84770665"
} }
}, }
{ },
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "containerProperties", "aspectName": "containerProperties",
"aspect": { "aspect": {
"value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"some_instance_name\", \"database\": \"test-database\"}, \"name\": \"test-database\", \"qualifiedName\": \"arn:aws:glue:us-west-2:123412341234:database/test-database\"}", "json": {
"contentType": "application/json" "customProperties": {
} "platform": "glue",
"instance": "some_instance_name",
"database": "test-database"
}, },
{ "name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
}
}
},
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "status", "aspectName": "status",
"aspect": { "aspect": {
"value": "{\"removed\": false}", "json": {
"contentType": "application/json" "removed": false
} }
}, }
{ },
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "dataPlatformInstance", "aspectName": "dataPlatformInstance",
"aspect": { "aspect": {
"value": "{\"platform\": \"urn:li:dataPlatform:glue\", \"instance\": \"urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)\"}", "json": {
"contentType": "application/json" "platform": "urn:li:dataPlatform:glue",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)"
} }
}, }
{ },
{
"entityType": "container", "entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"Database\"]}", "json": {
"contentType": "application/json" "typeNames": [
"Database"
]
} }
}, }
{ },
{
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_jsons_markers,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_jsons_markers,PROD)",
@ -459,28 +489,32 @@
] ]
} }
} }
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_jsons_markers,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_jsons_markers,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"table\"]}", "json": {
"contentType": "application/json" "typeNames": [
"table"
]
} }
}, }
{ },
{
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_jsons_markers,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_jsons_markers,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:9fb26491b2c92dde9e80791dbecca9ca\"}", "json": {
"contentType": "application/json" "container": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca"
} }
}, }
{ },
{
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)",
@ -640,28 +674,32 @@
] ]
} }
} }
}, },
{ {
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "subTypes", "aspectName": "subTypes",
"aspect": { "aspect": {
"value": "{\"typeNames\": [\"table\"]}", "json": {
"contentType": "application/json" "typeNames": [
"table"
]
} }
}, }
{ },
{
"entityType": "dataset", "entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)",
"changeType": "UPSERT", "changeType": "UPSERT",
"aspectName": "container", "aspectName": "container",
"aspect": { "aspect": {
"value": "{\"container\": \"urn:li:container:9fb26491b2c92dde9e80791dbecca9ca\"}", "json": {
"contentType": "application/json" "container": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca"
} }
}, }
{ },
{
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": {
"urn": "urn:li:dataFlow:(glue,test-job-1,PROD)", "urn": "urn:li:dataFlow:(glue,test-job-1,PROD)",
@ -682,8 +720,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": {
"urn": "urn:li:dataFlow:(glue,test-job-2,PROD)", "urn": "urn:li:dataFlow:(glue,test-job-2,PROD)",
@ -704,8 +742,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)",
@ -737,8 +775,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform1_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform1_job1)",
@ -770,8 +808,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)",
@ -803,8 +841,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Join-Transform3_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Join-Transform3_job1)",
@ -837,8 +875,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)",
@ -868,8 +906,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform5_job1)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform5_job1)",
@ -899,8 +937,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)",
@ -924,8 +962,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SplitFields-Transform0_job2)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SplitFields-Transform0_job2)",
@ -959,8 +997,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)",
@ -992,8 +1030,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)",
@ -1025,8 +1063,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": {
"urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SelectFields-Transform3_job2)", "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SelectFields-Transform3_job2)",
@ -1060,8 +1098,8 @@
] ]
} }
} }
}, },
{ {
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)",
@ -1085,5 +1123,137 @@
] ]
} }
} }
},
{
"entityType": "dataFlow",
"entityUrn": "urn:li:dataFlow:(glue,test-job-1,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
} }
}
},
{
"entityType": "dataFlow",
"entityUrn": "urn:li:dataFlow:(glue,test-job-2,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform1_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform5_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Join-Transform3_job1)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SelectFields-Transform3_job2)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "dataJob",
"entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SplitFields-Transform0_job2)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
}
] ]

14
metadata-ingestion/tests/unit/test_glue_source.py
View File

@ -324,14 +324,10 @@ def test_glue_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph):
# part of the second state # part of the second state
state1 = cast(BaseSQLAlchemyCheckpointState, checkpoint1.state) state1 = cast(BaseSQLAlchemyCheckpointState, checkpoint1.state)
state2 = cast(BaseSQLAlchemyCheckpointState, checkpoint2.state) state2 = cast(BaseSQLAlchemyCheckpointState, checkpoint2.state)
difference_urns = list( difference_urns = set(
state1.get_urns_not_in(type="*", other_checkpoint_state=state2) state1.get_urns_not_in(type="*", other_checkpoint_state=state2)
) )
assert difference_urns == {
assert len(difference_urns) == 1 "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)",
"urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba",
urn1 = ( }
"urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)"
)
assert urn1 in difference_urns