yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-28 10:28:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): delta-lake - extract table history into operation aspect (#5277)

Browse Source 
Co-authored-by: Shirshanka Das <shirshanka@apache.org>
This commit is contained in:
Mugdha Hardikar 2022-08-08 00:05:09 +05:30 committed by GitHub
parent da0258c6ad
commit b32a0723a1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 828 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py
View File

@ -59,6 +59,10 @@ class DeltaLakeSourceConfig(PlatformSourceConfigBase, EnvBasedSourceConfigBase):
default=AllowDenyPattern.allow_all(),
description="regex patterns for tables to filter in ingestion.",
)
version_history_lookback: Optional[int] = Field(
default=1,
description="Number of previous version histories to be ingested. Defaults to 1. If set to -1 all version history will be ingested.",
)
s3: Optional[S3] = Field()
@ -72,14 +76,20 @@ class DeltaLakeSourceConfig(PlatformSourceConfigBase, EnvBasedSourceConfigBase):
def get_complete_path(self):
return self._complete_path
@pydantic.validator("version_history_lookback")
def negative_version_history_implies_no_limit(cls, v):
if v and v < 0:
return None
return v
@pydantic.root_validator()
def validate_config(cls, values: Dict) -> Dict[str, Any]:
values["_is_s3"] = is_s3_uri(values["base_path"])
values["_is_s3"] = is_s3_uri(values.get("base_path", ""))
if values["_is_s3"]:
if values["s3"] is None:
if values.get("s3") is None:
raise ValueError("s3 config must be set for s3 path")
values["_complete_path"] = values["base_path"]
if values["relative_path"] is not None:
values["_complete_path"] = values.get("base_path")
if values.get("relative_path") is not None:
values[
"_complete_path"
] = f"{values['_complete_path'].rstrip('/')}/{values['relative_path'].lstrip('/')}"

4
metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py
View File

@ -24,11 +24,7 @@ def read_delta_table(
except PyDeltaTableError as e:
if "Not a Delta table" not in str(e):
import pdb
pdb.set_trace()
raise e
return delta_table

78
metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py
View File

@ -1,5 +1,7 @@
import logging
import os
import time
from datetime import datetime
from typing import Callable, Iterable, List
from deltalake import DeltaTable
@ -8,6 +10,7 @@ from datahub.emitter.mce_builder import (
make_data_platform_urn,
make_dataset_urn_with_platform_instance,
)
from datahub.emitter.mcp import MetadataChangeProposalWrapper
from datahub.ingestion.api.common import PipelineContext, WorkUnit
from datahub.ingestion.api.decorators import (
SourceCapability,
@ -41,8 +44,11 @@ from datahub.metadata.com.linkedin.pegasus2avro.schema import (
SchemaMetadata,
)
from datahub.metadata.schema_classes import (
ChangeTypeClass,
DatasetPropertiesClass,
NullTypeClass,
OperationClass,
OperationTypeClass,
OtherSchemaClass,
)
from datahub.telemetry import telemetry
@ -54,6 +60,19 @@ config_options_to_report = [
"platform",
]
OPERATION_STATEMENT_TYPES = {
"INSERT": OperationTypeClass.INSERT,
"UPDATE": OperationTypeClass.UPDATE,
"DELETE": OperationTypeClass.DELETE,
"MERGE": OperationTypeClass.UPDATE,
"CREATE": OperationTypeClass.CREATE,
"CREATE_TABLE_AS_SELECT": OperationTypeClass.CREATE,
"CREATE_SCHEMA": OperationTypeClass.CREATE,
"DROP_TABLE": OperationTypeClass.DROP,
"REPLACE TABLE AS SELECT": OperationTypeClass.UPDATE,
"COPY INTO": OperationTypeClass.UPDATE,
}
@platform_name("Delta Lake", id="delta-lake")
@config_class(DeltaLakeSourceConfig)
@ -122,6 +141,58 @@ class DeltaLakeSource(Source):
return fields
def _create_operation_aspect_wu(
self, delta_table: DeltaTable, dataset_urn: str
) -> Iterable[MetadataWorkUnit]:
for hist in delta_table.history(
limit=self.source_config.version_history_lookback
):
# History schema picked up from https://docs.delta.io/latest/delta-utility.html#retrieve-delta-table-history
reported_time: int = int(time.time() * 1000)
last_updated_timestamp: int = hist["timestamp"]
statement_type = OPERATION_STATEMENT_TYPES.get(
hist.get("operation"), OperationTypeClass.CUSTOM
)
custom_type = (
hist.get("operation")
if statement_type == OperationTypeClass.CUSTOM
else None
)
operation_custom_properties = dict()
for key, val in hist.items():
if val is not None:
if isinstance(val, dict):
for k, v in val:
if v is not None:
operation_custom_properties[f"{key}_{k}"] = str(v)
else:
operation_custom_properties[key] = str(val)
operation_custom_properties.pop("timestamp", None)
operation_custom_properties.pop("operation", None)
operation_aspect = OperationClass(
timestampMillis=reported_time,
lastUpdatedTimestamp=last_updated_timestamp,
operationType=statement_type,
customOperationType=custom_type,
customProperties=operation_custom_properties,
)
mcp = MetadataChangeProposalWrapper(
entityType="dataset",
aspectName="operation",
changeType=ChangeTypeClass.UPSERT,
entityUrn=dataset_urn,
aspect=operation_aspect,
)
operational_wu = MetadataWorkUnit(
id=f"{datetime.fromtimestamp(last_updated_timestamp / 1000).isoformat()}-operation-aspect-{dataset_urn}",
mcp=mcp,
)
self.report.report_workunit(operational_wu)
yield operational_wu
def ingest_table(
self, delta_table: DeltaTable, path: str
) -> Iterable[MetadataWorkUnit]:
@ -164,11 +235,6 @@ class DeltaLakeSource(Source):
"version": str(delta_table.version()),
"location": self.source_config.get_complete_path(),
}
customProperties.update(delta_table.history()[-1])
customProperties["version_creation_time"] = customProperties["timestamp"]
del customProperties["timestamp"]
for key in customProperties.keys():
customProperties[key] = str(customProperties[key])
dataset_properties = DatasetPropertiesClass(
description=delta_table.metadata().description,
@ -221,6 +287,8 @@ class DeltaLakeSource(Source):
self.report.report_workunit(wu)
yield wu
yield from self._create_operation_aspect_wu(delta_table, dataset_urn)
def process_folder(
self, path: str, get_folders: Callable[[str], Iterable[str]]
) -> Iterable[MetadataWorkUnit]:

343
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json
View File

@ -13,15 +13,7 @@
"table_creation_time": "1655831648843",
"id": "de711767-c7b9-4c33-99d7-510978dc1fa5",
"version": "4",
"location": "tests/integration/delta_lake/test_data/delta_tables",
"operation": "WRITE",
"operationParameters": "{}",
"readVersion": "3",
"isolationLevel": "Serializable",
"isBlindAppend": "True",
"operationMetrics": "{}",
"engineInfo": "local Delta-Standalone/0.4.0",
"version_creation_time": "1655831649788"
"location": "tests/integration/delta_lake/test_data/delta_tables"
},
"externalUrl": null,
"name": "my_table_no_name",
@ -514,6 +506,101 @@
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649166}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649715}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649731}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649754}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649788}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
@ -528,14 +615,7 @@
"table_creation_time": "1655664813952",
"id": "eca9d2a0-4ce6-4ace-a732-75fda0157fb8",
"version": "0",
"location": "tests/integration/delta_lake/test_data/delta_tables",
"operation": "CONVERT",
"operationParameters": "{}",
"isolationLevel": "Serializable",
"isBlindAppend": "True",
"operationMetrics": "{}",
"engineInfo": "local Delta-Standalone/0.4.0",
"version_creation_time": "1655664815399"
"location": "tests/integration/delta_lake/test_data/delta_tables"
},
"externalUrl": null,
"name": "my_table",
@ -727,6 +807,25 @@
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/sales,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"CONVERT\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655664815399}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
@ -741,15 +840,7 @@
"table_creation_time": "1655831476360",
"id": "628d06df-ecb0-4314-a97e-75d8872db7c3",
"version": "4",
"location": "tests/integration/delta_lake/test_data/delta_tables",
"operation": "WRITE",
"operationParameters": "{}",
"readVersion": "3",
"isolationLevel": "Serializable",
"isBlindAppend": "True",
"operationMetrics": "{}",
"engineInfo": "local Delta-Standalone/0.4.0",
"version_creation_time": "1655831477768"
"location": "tests/integration/delta_lake/test_data/delta_tables"
},
"externalUrl": null,
"name": "test-table-basic",
@ -881,6 +972,101 @@
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831476907}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477701}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477726}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477745}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
@ -895,15 +1081,7 @@
"table_creation_time": "1655831864836",
"id": "3775a0fd-4f58-4dea-b71a-e3fedb10f5b4",
"version": "4",
"location": "tests/integration/delta_lake/test_data/delta_tables",
"operation": "WRITE",
"operationParameters": "{}",
"readVersion": "3",
"isolationLevel": "Serializable",
"isBlindAppend": "True",
"operationMetrics": "{}",
"engineInfo": "local Delta-Standalone/0.4.0",
"version_creation_time": "1655831866541"
"location": "tests/integration/delta_lake/test_data/delta_tables"
},
"externalUrl": null,
"name": "test-table-inner",
@ -1110,5 +1288,100 @@
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831865396}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866337}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866398}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866447}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866541}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
}
]

343
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json
View File

@ -13,15 +13,7 @@
"table_creation_time": "1655831648843",
"id": "de711767-c7b9-4c33-99d7-510978dc1fa5",
"version": "4",
"location": "tests/integration/delta_lake/test_data/delta_tables",
"operation": "WRITE",
"operationParameters": "{}",
"readVersion": "3",
"isolationLevel": "Serializable",
"isBlindAppend": "True",
"operationMetrics": "{}",
"engineInfo": "local Delta-Standalone/0.4.0",
"version_creation_time": "1655831649788"
"location": "tests/integration/delta_lake/test_data/delta_tables"
},
"externalUrl": null,
"name": "my_table_no_name",
@ -514,6 +506,101 @@
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649166}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649715}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649731}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649754}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649788}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
@ -528,14 +615,7 @@
"table_creation_time": "1655664813952",
"id": "eca9d2a0-4ce6-4ace-a732-75fda0157fb8",
"version": "0",
"location": "tests/integration/delta_lake/test_data/delta_tables",
"operation": "CONVERT",
"operationParameters": "{}",
"isolationLevel": "Serializable",
"isBlindAppend": "True",
"operationMetrics": "{}",
"engineInfo": "local Delta-Standalone/0.4.0",
"version_creation_time": "1655664815399"
"location": "tests/integration/delta_lake/test_data/delta_tables"
},
"externalUrl": null,
"name": "my_table",
@ -727,6 +807,25 @@
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/sales,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"CONVERT\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655664815399}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
@ -741,15 +840,7 @@
"table_creation_time": "1655831476360",
"id": "628d06df-ecb0-4314-a97e-75d8872db7c3",
"version": "4",
"location": "tests/integration/delta_lake/test_data/delta_tables",
"operation": "WRITE",
"operationParameters": "{}",
"readVersion": "3",
"isolationLevel": "Serializable",
"isBlindAppend": "True",
"operationMetrics": "{}",
"engineInfo": "local Delta-Standalone/0.4.0",
"version_creation_time": "1655831477768"
"location": "tests/integration/delta_lake/test_data/delta_tables"
},
"externalUrl": null,
"name": "test-table-basic",
@ -881,6 +972,101 @@
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831476907}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477701}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477726}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477745}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
@ -895,15 +1081,7 @@
"table_creation_time": "1655831864836",
"id": "3775a0fd-4f58-4dea-b71a-e3fedb10f5b4",
"version": "4",
"location": "tests/integration/delta_lake/test_data/delta_tables",
"operation": "WRITE",
"operationParameters": "{}",
"readVersion": "3",
"isolationLevel": "Serializable",
"isBlindAppend": "True",
"operationMetrics": "{}",
"engineInfo": "local Delta-Standalone/0.4.0",
"version_creation_time": "1655831866541"
"location": "tests/integration/delta_lake/test_data/delta_tables"
},
"externalUrl": null,
"name": "test-table-inner",
@ -1110,5 +1288,100 @@
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831865396}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866337}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866398}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866447}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866541}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
}
]

105
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json
View File

@ -13,15 +13,7 @@
"table_creation_time": "1655831476360",
"id": "628d06df-ecb0-4314-a97e-75d8872db7c3",
"version": "4",
"location": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic/",
"operation": "WRITE",
"operationParameters": "{}",
"readVersion": "3",
"isolationLevel": "Serializable",
"isBlindAppend": "True",
"operationMetrics": "{}",
"engineInfo": "local Delta-Standalone/0.4.0",
"version_creation_time": "1655831477768"
"location": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic/"
},
"externalUrl": null,
"name": "test-table-basic",
@ -209,5 +201,100 @@
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831476907}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477701}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477726}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477745}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
}
]

29
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json
View File

@ -13,15 +13,7 @@
"table_creation_time": "1655831476360",
"id": "628d06df-ecb0-4314-a97e-75d8872db7c3",
"version": "4",
"location": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic",
"operation": "WRITE",
"operationParameters": "{}",
"readVersion": "3",
"isolationLevel": "Serializable",
"isBlindAppend": "True",
"operationMetrics": "{}",
"engineInfo": "local Delta-Standalone/0.4.0",
"version_creation_time": "1655831477768"
"location": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic"
},
"externalUrl": null,
"name": "test-table-basic",
@ -513,5 +505,24 @@
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "operation",
"aspect": {
"value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json",
"registryName": null,
"registryVersion": null,
"properties": null
}
}
]

5
metadata-ingestion/tests/integration/delta_lake/sources/local/allow_table.json
View File

@ -6,6 +6,7 @@
"platform_instance": "my-platform",
"table_pattern": {
"allow": ["s*"]
}
},
"version_history_lookback": -1
}
}
}

5
metadata-ingestion/tests/integration/delta_lake/sources/local/inner_table.json
View File

@ -2,6 +2,7 @@
"type": "delta-lake",
"config": {
"env": "UAT",
"base_path": "tests/integration/delta_lake/test_data/delta_tables"
"base_path": "tests/integration/delta_lake/test_data/delta_tables",
"version_history_lookback": -1
}
}
}

3
metadata-ingestion/tests/integration/delta_lake/sources/local/relative_path.json
View File

@ -3,6 +3,7 @@
"config": {
"env": "UAT",
"base_path": "tests/integration/delta_lake/test_data/",
"relative_path":"delta_tables/my_table_basic/"
"relative_path":"delta_tables/my_table_basic/",
"version_history_lookback": -1
}
}

2
metadata-ingestion/tests/integration/delta_lake/sources/local/single_table.json
View File

@ -4,4 +4,4 @@
"env": "UAT",
"base_path": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic"
}
}
}