yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-11 08:54:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/delta-lake): support parsing nested types correctly (#9862)

Browse Source
This commit is contained in:
dushayntAW 2024-03-06 19:22:01 +05:30 committed by GitHub
parent 4fbe8146a4
commit 68a26b47b2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
56 changed files with 3423 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py
View File

@ -1,7 +1,8 @@
import json
import logging
import os
import time
from typing import Dict, Iterable, List
from typing import Any, Dict, Iterable, List
from urllib.parse import urlparse
from deltalake import DeltaTable
@ -35,23 +36,22 @@ from datahub.ingestion.source.delta_lake.delta_lake_utils import (
read_delta_table,
)
from datahub.ingestion.source.delta_lake.report import DeltaLakeSourceReport
from datahub.ingestion.source.schema_inference.csv_tsv import tableschema_type_map
from datahub.metadata._schema_classes import SchemaFieldClass
from datahub.metadata.com.linkedin.pegasus2avro.common import Status
from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
from datahub.metadata.com.linkedin.pegasus2avro.schema import (
SchemaField,
SchemaFieldDataType,
SchemaMetadata,
)
from datahub.metadata.schema_classes import (
DatasetPropertiesClass,
NullTypeClass,
OperationClass,
OperationTypeClass,
OtherSchemaClass,
)
from datahub.telemetry import telemetry
from datahub.utilities.hive_schema_to_avro import get_schema_fields_for_hive_column
logging.getLogger("py4j").setLevel(logging.ERROR)
logger: logging.Logger = logging.getLogger(__name__)
@ -126,26 +126,57 @@ class DeltaLakeSource(Source):
config = DeltaLakeSourceConfig.parse_obj(config_dict)
return cls(config, ctx)
def delta_type_to_hive_type(self, field_type: Any) -> str:
if isinstance(field_type, str):
"""
return the field type
"""
return field_type
else:
if field_type.get("type") == "array":
"""
if array is of complex type, recursively parse the
fields and create the native datatype
"""
return (
"array<"
+ self.delta_type_to_hive_type(field_type.get("elementType"))
+ ">"
)
elif field_type.get("type") == "struct":
parsed_struct = ""
for field in field_type.get("fields"):
"""
if field is of complex type, recursively parse
and create the native datatype
"""
parsed_struct += (
"{0}:{1}".format(
field.get("name"),
self.delta_type_to_hive_type(field.get("type")),
)
+ ","
)
return "struct<" + parsed_struct.rstrip(",") + ">"
return ""
def _parse_datatype(self, raw_field_json_str: str) -> List[SchemaFieldClass]:
raw_field_json = json.loads(raw_field_json_str)
# get the parent field name and type
field_name = raw_field_json.get("name")
field_type = self.delta_type_to_hive_type(raw_field_json.get("type"))
return get_schema_fields_for_hive_column(field_name, field_type)
def get_fields(self, delta_table: DeltaTable) -> List[SchemaField]:
fields: List[SchemaField] = []
for raw_field in delta_table.schema().fields:
field = SchemaField(
fieldPath=raw_field.name,
type=SchemaFieldDataType(
tableschema_type_map.get(raw_field.type.type, NullTypeClass)()
),
nativeDataType=raw_field.type.type,
recursive=False,
nullable=raw_field.nullable,
description=str(raw_field.metadata),
isPartitioningKey=True
if raw_field.name in delta_table.metadata().partition_columns
else False,
)
fields.append(field)
fields = sorted(fields, key=lambda f: f.fieldPath)
parsed_data_list = self._parse_datatype(raw_field.to_json())
fields = fields + parsed_data_list
fields = sorted(fields, key=lambda f: f.fieldPath)
return fields
def _create_operation_aspect_wu(

180
metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json
View File

@ -45,79 +45,21 @@
},
"fields": [
{
"fieldPath": "customer",
"fieldPath": "[version=2.0].[type=float].total_cost",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
},
{
"fieldPath": "day",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
},
{
"fieldPath": "month",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
},
{
"fieldPath": "sale_id",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
},
{
"fieldPath": "total_cost",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NullType": {}
}
},
"nativeDataType": "float",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}"
},
{
"fieldPath": "year",
"fieldPath": "[version=2.0].[type=int].day",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -126,7 +68,59 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "[version=2.0].[type=int].month",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "[version=2.0].[type=int].year",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "[version=2.0].[type=string].customer",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
},
{
"fieldPath": "[version=2.0].[type=string].sale_id",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
}
]
}
@ -380,6 +374,68 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-test-bucket/delta_tables/sales,DEV)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd",
"urn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd"
},
{
"id": "urn:li:container:acebf8bcf966274632d3d2b710ef4947",
"urn": "urn:li:container:acebf8bcf966274632d3d2b710ef4947"
}
]
}
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": []
}
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:acebf8bcf966274632d3d2b710ef4947",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd",
"urn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd"
}
]
}
},
"systemMetadata": {
"lastObserved": 1672531200000,
"runId": "delta-lake-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-test-bucket/delta_tables/sales,DEV)",

475
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json
View File

@ -45,9 +45,8 @@
},
"fields": [
{
"fieldPath": "bar",
"fieldPath": "[version=2.0].[type=int].foo",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -56,12 +55,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "foo",
"fieldPath": "[version=2.0].[type=int].bar",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -70,12 +68,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "zip",
"fieldPath": "[version=2.0].[type=string].zip",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
@ -84,7 +81,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
}
]
}
@ -924,23 +921,8 @@
},
"fields": [
{
"fieldPath": "customer",
"fieldPath": "[version=2.0].[type=int].year",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
},
{
"fieldPath": "day",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -949,12 +931,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "month",
"fieldPath": "[version=2.0].[type=int].month",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -963,12 +944,24 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "sale_id",
"fieldPath": "[version=2.0].[type=int].day",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "[version=2.0].[type=string].sale_id",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
@ -977,35 +970,33 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
},
{
"fieldPath": "total_cost",
"fieldPath": "[version=2.0].[type=string].customer",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NullType": {}
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
},
{
"fieldPath": "[version=2.0].[type=float].total_cost",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "float",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
},
{
"fieldPath": "year",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}"
}
]
}
@ -1150,9 +1141,8 @@
},
"fields": [
{
"fieldPath": "bar",
"fieldPath": "[version=2.0].[type=int].foo",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -1161,12 +1151,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "foo",
"fieldPath": "[version=2.0].[type=int].bar",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -1175,12 +1164,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "zip",
"fieldPath": "[version=2.0].[type=string].zip",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
@ -1189,7 +1177,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
}
]
}
@ -1455,9 +1443,8 @@
},
"fields": [
{
"fieldPath": "bar",
"fieldPath": "[version=2.0].[type=int].foo",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -1466,12 +1453,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "foo",
"fieldPath": "[version=2.0].[type=int].bar",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -1480,12 +1466,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "zip",
"fieldPath": "[version=2.0].[type=string].zip",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
@ -1494,7 +1479,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
}
]
}
@ -1803,6 +1788,360 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)",
"urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)"
},
{
"id": "urn:li:container:189046201d696e7810132cfa64dad337",
"urn": "urn:li:container:189046201d696e7810132cfa64dad337"
},
{
"id": "urn:li:container:acf0f3806f475a7397ee745329ef2967",
"urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967"
},
{
"id": "urn:li:container:1876d057d0ee364677b85427342e2c82",
"urn": "urn:li:container:1876d057d0ee364677b85427342e2c82"
},
{
"id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4",
"urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4"
},
{
"id": "urn:li:container:a282913be26fceff334523c2be119df1",
"urn": "urn:li:container:a282913be26fceff334523c2be119df1"
},
{
"id": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef",
"urn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)",
"urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)",
"urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)"
},
{
"id": "urn:li:container:189046201d696e7810132cfa64dad337",
"urn": "urn:li:container:189046201d696e7810132cfa64dad337"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)",
"urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)"
},
{
"id": "urn:li:container:189046201d696e7810132cfa64dad337",
"urn": "urn:li:container:189046201d696e7810132cfa64dad337"
},
{
"id": "urn:li:container:acf0f3806f475a7397ee745329ef2967",
"urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)",
"urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)"
},
{
"id": "urn:li:container:189046201d696e7810132cfa64dad337",
"urn": "urn:li:container:189046201d696e7810132cfa64dad337"
},
{
"id": "urn:li:container:acf0f3806f475a7397ee745329ef2967",
"urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967"
},
{
"id": "urn:li:container:1876d057d0ee364677b85427342e2c82",
"urn": "urn:li:container:1876d057d0ee364677b85427342e2c82"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:a282913be26fceff334523c2be119df1",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)",
"urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)"
},
{
"id": "urn:li:container:189046201d696e7810132cfa64dad337",
"urn": "urn:li:container:189046201d696e7810132cfa64dad337"
},
{
"id": "urn:li:container:acf0f3806f475a7397ee745329ef2967",
"urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967"
},
{
"id": "urn:li:container:1876d057d0ee364677b85427342e2c82",
"urn": "urn:li:container:1876d057d0ee364677b85427342e2c82"
},
{
"id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4",
"urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)",
"urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)"
},
{
"id": "urn:li:container:189046201d696e7810132cfa64dad337",
"urn": "urn:li:container:189046201d696e7810132cfa64dad337"
},
{
"id": "urn:li:container:acf0f3806f475a7397ee745329ef2967",
"urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967"
},
{
"id": "urn:li:container:1876d057d0ee364677b85427342e2c82",
"urn": "urn:li:container:1876d057d0ee364677b85427342e2c82"
},
{
"id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4",
"urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4"
},
{
"id": "urn:li:container:a282913be26fceff334523c2be119df1",
"urn": "urn:li:container:a282913be26fceff334523c2be119df1"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/sales,UAT)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)",
"urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)"
},
{
"id": "urn:li:container:189046201d696e7810132cfa64dad337",
"urn": "urn:li:container:189046201d696e7810132cfa64dad337"
},
{
"id": "urn:li:container:acf0f3806f475a7397ee745329ef2967",
"urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967"
},
{
"id": "urn:li:container:1876d057d0ee364677b85427342e2c82",
"urn": "urn:li:container:1876d057d0ee364677b85427342e2c82"
},
{
"id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4",
"urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4"
},
{
"id": "urn:li:container:a282913be26fceff334523c2be119df1",
"urn": "urn:li:container:a282913be26fceff334523c2be119df1"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)",
"urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)"
},
{
"id": "urn:li:container:189046201d696e7810132cfa64dad337",
"urn": "urn:li:container:189046201d696e7810132cfa64dad337"
},
{
"id": "urn:li:container:acf0f3806f475a7397ee745329ef2967",
"urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967"
},
{
"id": "urn:li:container:1876d057d0ee364677b85427342e2c82",
"urn": "urn:li:container:1876d057d0ee364677b85427342e2c82"
},
{
"id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4",
"urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4"
},
{
"id": "urn:li:container:a282913be26fceff334523c2be119df1",
"urn": "urn:li:container:a282913be26fceff334523c2be119df1"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)",
"urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)"
},
{
"id": "urn:li:container:189046201d696e7810132cfa64dad337",
"urn": "urn:li:container:189046201d696e7810132cfa64dad337"
},
{
"id": "urn:li:container:acf0f3806f475a7397ee745329ef2967",
"urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967"
},
{
"id": "urn:li:container:1876d057d0ee364677b85427342e2c82",
"urn": "urn:li:container:1876d057d0ee364677b85427342e2c82"
},
{
"id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4",
"urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4"
},
{
"id": "urn:li:container:a282913be26fceff334523c2be119df1",
"urn": "urn:li:container:a282913be26fceff334523c2be119df1"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "allow_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",

434
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json
View File

@ -45,9 +45,8 @@
},
"fields": [
{
"fieldPath": "bar",
"fieldPath": "[version=2.0].[type=int].foo",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -56,12 +55,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "foo",
"fieldPath": "[version=2.0].[type=int].bar",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -70,12 +68,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "zip",
"fieldPath": "[version=2.0].[type=string].zip",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
@ -84,7 +81,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
}
]
}
@ -889,23 +886,8 @@
},
"fields": [
{
"fieldPath": "customer",
"fieldPath": "[version=2.0].[type=int].year",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
},
{
"fieldPath": "day",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -914,12 +896,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "month",
"fieldPath": "[version=2.0].[type=int].month",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -928,12 +909,24 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "sale_id",
"fieldPath": "[version=2.0].[type=int].day",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "[version=2.0].[type=string].sale_id",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
@ -942,35 +935,33 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
},
{
"fieldPath": "total_cost",
"fieldPath": "[version=2.0].[type=string].customer",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NullType": {}
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
},
{
"fieldPath": "[version=2.0].[type=float].total_cost",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "float",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
},
{
"fieldPath": "year",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}"
}
]
}
@ -1111,9 +1102,8 @@
},
"fields": [
{
"fieldPath": "bar",
"fieldPath": "[version=2.0].[type=int].foo",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -1122,12 +1112,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "foo",
"fieldPath": "[version=2.0].[type=int].bar",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -1136,12 +1125,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "zip",
"fieldPath": "[version=2.0].[type=string].zip",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
@ -1150,7 +1138,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
}
]
}
@ -1412,9 +1400,8 @@
},
"fields": [
{
"fieldPath": "bar",
"fieldPath": "[version=2.0].[type=int].foo",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -1423,12 +1410,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "foo",
"fieldPath": "[version=2.0].[type=int].bar",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -1437,12 +1423,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "zip",
"fieldPath": "[version=2.0].[type=string].zip",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
@ -1451,7 +1436,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
}
]
}
@ -1754,6 +1739,319 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
},
{
"id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6"
},
{
"id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4",
"urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4"
},
{
"id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab",
"urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab"
},
{
"id": "urn:li:container:6bb6dc6de93177210067d00b45b481bb",
"urn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": []
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
},
{
"id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
},
{
"id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6"
},
{
"id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4",
"urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
},
{
"id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6"
},
{
"id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4",
"urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4"
},
{
"id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab",
"urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/sales,UAT)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
},
{
"id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6"
},
{
"id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4",
"urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4"
},
{
"id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab",
"urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
},
{
"id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6"
},
{
"id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4",
"urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4"
},
{
"id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab",
"urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
},
{
"id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6"
},
{
"id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4",
"urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4"
},
{
"id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab",
"urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "inner_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)",

52
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json
View File

@ -45,9 +45,8 @@
},
"fields": [
{
"fieldPath": "bar",
"fieldPath": "[version=2.0].[type=int].foo",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -56,12 +55,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "foo",
"fieldPath": "[version=2.0].[type=int].bar",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -70,12 +68,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "zip",
"fieldPath": "[version=2.0].[type=string].zip",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
@ -84,7 +81,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
}
]
}
@ -350,6 +347,43 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:85267d161e1a2ffa647cec6c1188549f",
"urn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": []
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "relative_path.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)",

176
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json
View File

@ -44,9 +44,8 @@
},
"fields": [
{
"fieldPath": "bar",
"fieldPath": "[version=2.0].[type=int].foo",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -55,12 +54,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "foo",
"fieldPath": "[version=2.0].[type=int].bar",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -69,12 +67,11 @@
"nativeDataType": "integer",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": true
"jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}"
},
{
"fieldPath": "zip",
"fieldPath": "[version=2.0].[type=string].zip",
"nullable": true,
"description": "{}",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
@ -83,7 +80,7 @@
"nativeDataType": "string",
"recursive": false,
"isPartOfKey": false,
"isPartitioningKey": false
"jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}"
}
]
}
@ -686,6 +683,167 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
},
{
"id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6"
},
{
"id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4",
"urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4"
},
{
"id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab",
"urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": []
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
},
{
"id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf",
"urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf"
},
{
"id": "urn:li:container:974a39dc631803eddedc699cc9bb9759",
"urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759"
},
{
"id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6",
"urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6"
},
{
"id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4",
"urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "single_table.json",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)",

2231
metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json Normal file
View File

File diff suppressed because it is too large Load Diff

11
metadata-ingestion/tests/integration/delta_lake/sources/local/tables_with_nested_datatypes.json Normal file
View File

@ -0,0 +1,11 @@
{
"type": "delta-lake",
"config": {
"env": "UAT",
"base_path": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype",
"platform_instance": "my-platform",
"table_pattern": {
"allow": ["s*"]
}
}
}

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/_delta_log/.00000000000000000000.json.crc Normal file
View File

Binary file not shown.

5
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/_delta_log/00000000000000000000.json Normal file
View File

@ -0,0 +1,5 @@
{"commitInfo":{"timestamp":1709110542636,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"3473"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"e78cea2a-7fd7-4aab-9b50-776e7638fb54"}}
{"metaData":{"id":"73cae4a6-3988-4337-89ca-af58dd528b0b","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1709110539186}}
{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
{"add":{"path":"part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet","partitionValues":{},"size":1380,"modificationTime":1709110541918,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row1\",\"data\":{\"_1\":\"value1\",\"_2\":{\"_1\":20,\"_2\":\"test\"}}},\"maxValues\":{\"id\":\"row1\",\"data\":{\"_1\":\"value1\",\"_2\":{\"_1\":20,\"_2\":\"test\"}}},\"nullCount\":{\"id\":0,\"data\":{\"_1\":0,\"_2\":{\"_1\":0,\"_2\":0}}}}"}}
{"add":{"path":"part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet","partitionValues":{},"size":1380,"modificationTime":1709110541918,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row2\",\"data\":{\"_1\":\"value2\",\"_2\":{\"_1\":30,\"_2\":\"test\"}}},\"maxValues\":{\"id\":\"row2\",\"data\":{\"_1\":\"value2\",\"_2\":{\"_1\":30,\"_2\":\"test\"}}},\"nullCount\":{\"id\":0,\"data\":{\"_1\":0,\"_2\":{\"_1\":0,\"_2\":0}}}}"}}

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/.00000000000000000000.json.crc Normal file
View File

Binary file not shown.

5
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/00000000000000000000.json Normal file
View File

@ -0,0 +1,5 @@
{"commitInfo":{"timestamp":1709535906725,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"3473"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"95fab5d1-7811-4326-96d2-6244adace432"}}
{"metaData":{"id":"363c76ca-3357-48c6-b14d-71262be23dbc","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"data\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1709535902908}}
{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
{"add":{"path":"part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet","partitionValues":{},"size":1380,"modificationTime":1709535905678,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":\"value1\",\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"maxValues\":{\"data\":{\"_1\":\"value1\",\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"nullCount\":{\"data\":{\"_1\":0,\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}}
{"add":{"path":"part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet","partitionValues":{},"size":1380,"modificationTime":1709535905678,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":\"value2\",\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"maxValues\":{\"data\":{\"_1\":\"value2\",\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"nullCount\":{\"data\":{\"_1\":0,\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}}

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/.00000000000000000000.json.crc Normal file
View File

Binary file not shown.

5
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/00000000000000000000.json Normal file
View File

@ -0,0 +1,5 @@
{"commitInfo":{"timestamp":1709536366367,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"4292"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"0ead04f5-e706-42f1-bae9-4829f2001edb"}}
{"metaData":{"id":"58b63865-38fa-4478-98cf-3a7fd3425e24","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"data\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1709536362734}}
{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
{"add":{"path":"part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet","partitionValues":{},"size":1710,"modificationTime":1709536365577,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"maxValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"nullCount\":{\"data\":{\"_1\":{\"_1\":0,\"_2\":0},\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}}
{"add":{"path":"part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet","partitionValues":{},"size":1710,"modificationTime":1709536365576,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"maxValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"nullCount\":{\"data\":{\"_1\":{\"_1\":0,\"_2\":0},\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}}

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/.00000000000000000000.json.crc Normal file
View File

Binary file not shown.

5
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/00000000000000000000.json Normal file
View File

@ -0,0 +1,5 @@
{"commitInfo":{"timestamp":1708329078869,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"2027"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"c1497bc8-403a-451f-b651-adeaad02a74d"}}
{"metaData":{"id":"bccc302c-006d-4de9-b572-0c1939e64fc5","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":{\"type\":\"array\",\"elementType\":\"string\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1708329075098}}
{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
{"add":{"path":"part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet","partitionValues":{},"size":779,"modificationTime":1708329077928,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row1\"},\"maxValues\":{\"id\":\"row1\"},\"nullCount\":{\"id\":0,\"data\":0}}"}}
{"add":{"path":"part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet","partitionValues":{},"size":779,"modificationTime":1708329077928,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row2\"},\"maxValues\":{\"id\":\"row2\"},\"nullCount\":{\"id\":0,\"data\":0}}"}}

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/.00000000000000000000.json.crc Normal file
View File

Binary file not shown.

5
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/00000000000000000000.json Normal file
View File

@ -0,0 +1,5 @@
{"commitInfo":{"timestamp":1708329897384,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"2943"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"dd7ddb85-1b3a-4f18-bd8f-1466adfe66b0"}}
{"metaData":{"id":"6f3b01e4-ae1e-415a-979a-34861346809b","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1708329893422}}
{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
{"add":{"path":"part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet","partitionValues":{},"size":1149,"modificationTime":1708329896608,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row1\"},\"maxValues\":{\"id\":\"row1\"},\"nullCount\":{\"id\":0,\"data\":0}}"}}
{"add":{"path":"part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet","partitionValues":{},"size":1153,"modificationTime":1708329896608,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row2\"},\"maxValues\":{\"id\":\"row2\"},\"nullCount\":{\"id\":0,\"data\":0}}"}}

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet.crc Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/.00000000000000000000.json.crc Normal file
View File

Binary file not shown.

5
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/00000000000000000000.json Normal file
View File

@ -0,0 +1,5 @@
{"commitInfo":{"timestamp":1708330178404,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"2773"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"96c8e2d9-55f7-416d-84de-55d01936e58e"}}
{"metaData":{"id":"9b656138-6370-412a-826b-4f20e659d5c2","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"long\",\"containsNull\":true},\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1708330174792}}
{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
{"add":{"path":"part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet","partitionValues":{},"size":1074,"modificationTime":1708330177598,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row1\"},\"maxValues\":{\"id\":\"row1\"},\"nullCount\":{\"id\":0,\"data\":0}}"}}
{"add":{"path":"part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet","partitionValues":{},"size":1074,"modificationTime":1708330177598,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row2\"},\"maxValues\":{\"id\":\"row2\"},\"nullCount\":{\"id\":0,\"data\":0}}"}}

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet Normal file
View File

Binary file not shown.

BIN
metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet Normal file
View File

Binary file not shown.