yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-22 16:18:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(ingest/delta-lake): Fix integration tests (#9056)

Browse Source
This commit is contained in:
Andrew Sikowitz 2023-10-20 01:40:23 -04:00 committed by GitHub
parent b1abd38a6b
commit 35d7770462
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 1173 additions and 947 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json
View File

@ -136,7 +136,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -156,7 +157,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -171,7 +173,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -186,7 +189,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -203,7 +207,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -218,7 +223,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -238,7 +244,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -253,7 +260,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -268,7 +276,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -285,7 +294,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -300,7 +310,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -320,7 +331,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -335,7 +347,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -355,14 +368,16 @@
"customProperties": {
"engineInfo": "local Delta-Standalone/0.4.0",
"isBlindAppend": "True",
"isolationLevel": "Serializable"
"isolationLevel": "Serializable",
"version": "0"
},
"lastUpdatedTimestamp": 1655664815399
}
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -386,7 +401,8 @@
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test"
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
}
]

855
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json
View File

File diff suppressed because it is too large Load Diff

855
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json
View File

File diff suppressed because it is too large Load Diff

260
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json
View File

@ -94,7 +94,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -114,7 +115,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -129,7 +131,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -144,7 +147,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -161,7 +165,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -176,7 +181,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -191,34 +197,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1615443388097,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "CUSTOM",
"customOperationType": "WRITE",
"customProperties": {
"engineInfo": "local Delta-Standalone/0.4.0",
"isBlindAppend": "True",
"isolationLevel": "Serializable"
},
"lastUpdatedTimestamp": 1655831476907
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -239,98 +219,135 @@
"engineInfo": "local Delta-Standalone/0.4.0",
"isBlindAppend": "True",
"isolationLevel": "Serializable",
"readVersion": "0"
},
"lastUpdatedTimestamp": 1655831477701
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1615443388097,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "CUSTOM",
"customOperationType": "WRITE",
"customProperties": {
"engineInfo": "local Delta-Standalone/0.4.0",
"isBlindAppend": "True",
"isolationLevel": "Serializable",
"readVersion": "1"
},
"lastUpdatedTimestamp": 1655831477726
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1615443388097,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "CUSTOM",
"customOperationType": "WRITE",
"customProperties": {
"engineInfo": "local Delta-Standalone/0.4.0",
"isBlindAppend": "True",
"isolationLevel": "Serializable",
"readVersion": "2"
},
"lastUpdatedTimestamp": 1655831477745
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1615443388097,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "CUSTOM",
"customOperationType": "WRITE",
"customProperties": {
"engineInfo": "local Delta-Standalone/0.4.0",
"isBlindAppend": "True",
"isolationLevel": "Serializable",
"readVersion": "3"
"readVersion": "3",
"version": "4"
},
"lastUpdatedTimestamp": 1655831477768
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1615443388097,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "CUSTOM",
"customOperationType": "WRITE",
"customProperties": {
"engineInfo": "local Delta-Standalone/0.4.0",
"isBlindAppend": "True",
"isolationLevel": "Serializable",
"readVersion": "2",
"version": "3"
},
"lastUpdatedTimestamp": 1655831477745
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1615443388097,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "CUSTOM",
"customOperationType": "WRITE",
"customProperties": {
"engineInfo": "local Delta-Standalone/0.4.0",
"isBlindAppend": "True",
"isolationLevel": "Serializable",
"readVersion": "1",
"version": "2"
},
"lastUpdatedTimestamp": 1655831477726
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1615443388097,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "CUSTOM",
"customOperationType": "WRITE",
"customProperties": {
"engineInfo": "local Delta-Standalone/0.4.0",
"isBlindAppend": "True",
"isolationLevel": "Serializable",
"readVersion": "0",
"version": "1"
},
"lastUpdatedTimestamp": 1655831477701
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"json": {
"timestampMillis": 1615443388097,
"partitionSpec": {
"type": "FULL_TABLE",
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "CUSTOM",
"customOperationType": "WRITE",
"customProperties": {
"engineInfo": "local Delta-Standalone/0.4.0",
"isBlindAppend": "True",
"isolationLevel": "Serializable",
"version": "0"
},
"lastUpdatedTimestamp": 1655831476907
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -350,7 +367,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json"
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
}
]

102
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json
View File

@ -93,7 +93,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -113,7 +114,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -128,7 +130,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -143,7 +146,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -160,7 +164,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -175,7 +180,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -195,7 +201,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -210,7 +217,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -225,7 +233,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -242,7 +251,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -257,7 +267,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -277,7 +288,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -297,7 +309,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -312,7 +325,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -327,7 +341,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -344,7 +359,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -359,7 +375,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -383,7 +400,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -403,7 +421,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -418,7 +437,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -433,7 +453,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -450,7 +471,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -465,7 +487,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -493,7 +516,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -513,7 +537,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -528,7 +553,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -543,7 +569,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -560,7 +587,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -575,7 +603,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -607,7 +636,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -622,7 +652,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -643,14 +674,16 @@
"engineInfo": "local Delta-Standalone/0.4.0",
"isBlindAppend": "True",
"isolationLevel": "Serializable",
"readVersion": "3"
"readVersion": "3",
"version": "4"
},
"lastUpdatedTimestamp": 1655831477768
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
@ -686,7 +719,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json"
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
}
]