test(ingest/delta-lake): Fix integration tests (#9056)

metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json

@@ -136,7 +136,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -156,7 +157,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -171,7 +173,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -186,7 +189,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -203,7 +207,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -218,7 +223,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -238,7 +244,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -253,7 +260,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -268,7 +276,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -285,7 +294,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -300,7 +310,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -320,7 +331,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -335,7 +347,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -355,14 +368,16 @@
             "customProperties": {
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
+                "isolationLevel": "Serializable",
+                "version": "0"
             },
             "lastUpdatedTimestamp": 1655664815399
         }
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -386,7 +401,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1672531200000,
-        "runId": "delta-lake-test"
+        "runId": "delta-lake-test",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]

metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json

@@ -94,7 +94,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -114,7 +115,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -129,7 +131,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -144,7 +147,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -161,7 +165,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -176,7 +181,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -191,34 +197,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
-    }
+        "lastRunId": "no-run-id-provided"
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1615443388097,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "operationType": "CUSTOM",
-            "customOperationType": "WRITE",
-            "customProperties": {
-                "engineInfo": "local Delta-Standalone/0.4.0",
-                "isBlindAppend": "True",
-                "isolationLevel": "Serializable"
-            },
-            "lastUpdatedTimestamp": 1655831476907
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
     }
 },
 {
@@ -239,98 +219,135 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "0"
+                "readVersion": "3",
-            },
+                "version": "4"
-            "lastUpdatedTimestamp": 1655831477701
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1615443388097,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "operationType": "CUSTOM",
-            "customOperationType": "WRITE",
-            "customProperties": {
-                "engineInfo": "local Delta-Standalone/0.4.0",
-                "isBlindAppend": "True",
-                "isolationLevel": "Serializable",
-                "readVersion": "1"
-            },
-            "lastUpdatedTimestamp": 1655831477726
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1615443388097,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "operationType": "CUSTOM",
-            "customOperationType": "WRITE",
-            "customProperties": {
-                "engineInfo": "local Delta-Standalone/0.4.0",
-                "isBlindAppend": "True",
-                "isolationLevel": "Serializable",
-                "readVersion": "2"
-            },
-            "lastUpdatedTimestamp": 1655831477745
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
-    }
-},
-{
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
-    "changeType": "UPSERT",
-    "aspectName": "operation",
-    "aspect": {
-        "json": {
-            "timestampMillis": 1615443388097,
-            "partitionSpec": {
-                "type": "FULL_TABLE",
-                "partition": "FULL_TABLE_SNAPSHOT"
-            },
-            "operationType": "CUSTOM",
-            "customOperationType": "WRITE",
-            "customProperties": {
-                "engineInfo": "local Delta-Standalone/0.4.0",
-                "isBlindAppend": "True",
-                "isolationLevel": "Serializable",
-                "readVersion": "3"
             },
             "lastUpdatedTimestamp": 1655831477768
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
+    "changeType": "UPSERT",
+    "aspectName": "operation",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1615443388097,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "operationType": "CUSTOM",
+            "customOperationType": "WRITE",
+            "customProperties": {
+                "engineInfo": "local Delta-Standalone/0.4.0",
+                "isBlindAppend": "True",
+                "isolationLevel": "Serializable",
+                "readVersion": "2",
+                "version": "3"
+            },
+            "lastUpdatedTimestamp": 1655831477745
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
+    "changeType": "UPSERT",
+    "aspectName": "operation",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1615443388097,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "operationType": "CUSTOM",
+            "customOperationType": "WRITE",
+            "customProperties": {
+                "engineInfo": "local Delta-Standalone/0.4.0",
+                "isBlindAppend": "True",
+                "isolationLevel": "Serializable",
+                "readVersion": "1",
+                "version": "2"
+            },
+            "lastUpdatedTimestamp": 1655831477726
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
+    "changeType": "UPSERT",
+    "aspectName": "operation",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1615443388097,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "operationType": "CUSTOM",
+            "customOperationType": "WRITE",
+            "customProperties": {
+                "engineInfo": "local Delta-Standalone/0.4.0",
+                "isBlindAppend": "True",
+                "isolationLevel": "Serializable",
+                "readVersion": "0",
+                "version": "1"
+            },
+            "lastUpdatedTimestamp": 1655831477701
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
+    "changeType": "UPSERT",
+    "aspectName": "operation",
+    "aspect": {
+        "json": {
+            "timestampMillis": 1615443388097,
+            "partitionSpec": {
+                "type": "FULL_TABLE",
+                "partition": "FULL_TABLE_SNAPSHOT"
+            },
+            "operationType": "CUSTOM",
+            "customOperationType": "WRITE",
+            "customProperties": {
+                "engineInfo": "local Delta-Standalone/0.4.0",
+                "isBlindAppend": "True",
+                "isolationLevel": "Serializable",
+                "version": "0"
+            },
+            "lastUpdatedTimestamp": 1655831476907
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1615443388097,
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -350,7 +367,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "relative_path.json"
+        "runId": "relative_path.json",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]

metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json

@@ -93,7 +93,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -113,7 +114,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -128,7 +130,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -143,7 +146,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -160,7 +164,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -175,7 +180,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -195,7 +201,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -210,7 +217,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -225,7 +233,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -242,7 +251,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -257,7 +267,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -277,7 +288,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -297,7 +309,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -312,7 +325,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -327,7 +341,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -344,7 +359,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -359,7 +375,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -383,7 +400,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -403,7 +421,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -418,7 +437,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -433,7 +453,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -450,7 +471,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -465,7 +487,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -493,7 +516,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -513,7 +537,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -528,7 +553,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -543,7 +569,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -560,7 +587,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -575,7 +603,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -607,7 +636,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -622,7 +652,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -643,14 +674,16 @@
                 "engineInfo": "local Delta-Standalone/0.4.0",
                 "isBlindAppend": "True",
                 "isolationLevel": "Serializable",
-                "readVersion": "3"
+                "readVersion": "3",
+                "version": "4"
             },
             "lastUpdatedTimestamp": 1655831477768
         }
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 },
 {
@@ -686,7 +719,8 @@
     },
     "systemMetadata": {
         "lastObserved": 1615443388097,
-        "runId": "single_table.json"
+        "runId": "single_table.json",
+        "lastRunId": "no-run-id-provided"
     }
 }
 ]