diff --git a/metadata-ingestion/examples/mce_files/bootstrap_mce.json b/metadata-ingestion/examples/mce_files/bootstrap_mce.json index d0e0075379..5452453932 100644 --- a/metadata-ingestion/examples/mce_files/bootstrap_mce.json +++ b/metadata-ingestion/examples/mce_files/bootstrap_mce.json @@ -1,3652 +1,4045 @@ [ - { - "auditHeader": null, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { - "urn": "urn:li:corpuser:jdoe", - "aspects": [ - { - "com.linkedin.pegasus2avro.identity.CorpUserInfo": { - "active": true, - "displayName": { - "string": "John Doe" - }, - "email": "jdoe@linkedin.com", - "title": { - "string": "Software Engineer" - }, - "managerUrn": null, - "departmentId": null, - "departmentName": null, - "firstName": null, - "lastName": null, - "fullName": { - "string": "John Doe" - }, - "countryCode": null - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { - "urn": "urn:li:corpGroup:jdoe", - "aspects": [ - { - "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { - "email": "jdoe@linkedin.com", - "admins": ["urn:li:corpuser:jdoe", "urn:li:corpuser:datahub"], - "members": ["urn:li:corpuser:jdoe", "urn:li:corpuser:datahub"], - "groups": [], - "slack": "bfoo-squad" - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { - "urn": "urn:li:corpGroup:bfoo", - "aspects": [ - { - "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { - "email": "bfoo@linkedin.com", - "admins": ["urn:li:corpuser:jdoe", "urn:li:corpuser:datahub"], - "members": ["urn:li:corpuser:jdoe", "urn:li:corpuser:datahub"], - "groups": ["urn:li:corpGroup:jdoe"], - "description": "This group is full of bfoos!" - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/prod/kafka/SampleKafkaDataset"] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": null, - "uri": null, - "tags": [], - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:jdoe", + "aspects": [ { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleKafkaSchema", - "platform": "urn:li:dataPlatform:kafka", - "version": 0, - "created": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", - "documentSchemaType": "AVRO" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=boolean].field_foo_2", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Foo field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "customProperties": {}, + "active": true, + "displayName": "John Doe", + "email": "jdoe@linkedin.com", + "title": "Software Engineer", + "fullName": "John Doe", + "system": false } - }, - "nativeDataType": "varchar(100)", - "globalTags": { - "tags": [{ "tag": "urn:li:tag:NeedsDocumentation" }] - }, - "recursive": false }, { - "fieldPath": "[version=2.0].[type=boolean].field_bar", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Bar field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} + "com.linkedin.pegasus2avro.common.Status": { + "removed": false } - }, - "nativeDataType": "boolean", - "recursive": false - }, + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741905, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { + "urn": "urn:li:corpGroup:jdoe", + "aspects": [ { - "fieldPath": "[version=2.0].[key=True].[type=int].id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id specifying which partition the message should go to" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} + "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { + "email": "jdoe@linkedin.com", + "admins": [ + "urn:li:corpuser:jdoe", + "urn:li:corpuser:datahub" + ], + "members": [ + "urn:li:corpuser:jdoe", + "urn:li:corpuser:datahub" + ], + "groups": [], + "slack": "bfoo-squad" } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/prod/hdfs/SampleHdfsDataset"] - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null }, { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { - "created": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "editableSchemaFieldInfo": [ - { - "fieldPath": "shipment_info", - "globalTags": { "tags": [{ "tag": "urn:li:tag:Legacy" }] } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHdfsSchema", - "platform": "urn:li:dataPlatform:hdfs", - "version": 0, - "created": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", - "documentSchemaType": "AVRO" - } - }, - "fields": [ - { - "fieldPath": "shipment_info", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} + "com.linkedin.pegasus2avro.common.Status": { + "removed": false } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741905, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { + "urn": "urn:li:corpGroup:bfoo", + "aspects": [ { - "fieldPath": "shipment_info.date", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info date description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { + "email": "bfoo@linkedin.com", + "admins": [ + "urn:li:corpuser:jdoe", + "urn:li:corpuser:datahub" + ], + "members": [ + "urn:li:corpuser:jdoe", + "urn:li:corpuser:datahub" + ], + "groups": [ + "urn:li:corpGroup:jdoe" + ], + "description": "This group is full of bfoos!" } - }, - "nativeDataType": "Date", - "recursive": false }, { - "fieldPath": "shipment_info.target", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info target description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.common.Status": { + "removed": false } - }, - "nativeDataType": "text", - "recursive": false - }, + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741906, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", + "aspects": [ { - "fieldPath": "shipment_info.destination", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info destination description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka/SampleKafkaDataset" + ] } - }, - "nativeDataType": "varchar(100)", - "recursive": false }, { - "fieldPath": "shipment_info.geo_info", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + }, + "tags": [] } - }, - "nativeDataType": "varchar(100)", - "recursive": false }, { - "fieldPath": "shipment_info.geo_info.lat", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info lat" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } } - }, - "nativeDataType": "float", - "recursive": false }, { - "fieldPath": "shipment_info.geo_info.lng", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Shipment info geo_info lng" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + ] } - }, - "nativeDataType": "float", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null }, { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", - "documentSchemaType": "AVRO" - } - }, - "fields": [ - { - "fieldPath": "field_foo", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Foo field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleKafkaSchema", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", + "documentSchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=boolean].field_foo_2", + "nullable": false, + "description": "Foo field description", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:NeedsDocumentation" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=boolean].field_bar", + "nullable": false, + "description": "Bar field description", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[key=True].[type=int].id", + "nullable": false, + "description": "Id specifying which partition the message should go to", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + } + ] } - }, - "nativeDataType": "varchar(100)", - "recursive": false, - "isPartOfKey": true }, { - "fieldPath": "field_bar", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Bar field description" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} + "com.linkedin.pegasus2avro.common.Status": { + "removed": false } - }, - "nativeDataType": "boolean", - "recursive": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Legacy" }] - } - } - ] - } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741907, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table where each row represents a single log event", - "qualifiedName": "a.b.c", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "aspects": [ { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", - "documentSchemaType": "AVRO" - } - }, - "fields": [ - { - "fieldPath": "event_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of your logging event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} + "com.linkedin.pegasus2avro.common.Status": { + "removed": false } - }, - "nativeDataType": "varchar(100)", - "recursive": false }, { - "fieldPath": "event_data", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Data of your event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/hdfs/SampleHdfsDataset" + ] } - }, - "nativeDataType": "boolean", - "recursive": false }, { - "fieldPath": "timestamp", - "jsonPath": null, - "nullable": false, - "description": { - "string": "TS the event was ingested" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } } - }, - "nativeDataType": "boolean", - "recursive": false }, { - "fieldPath": "browser", - "jsonPath": null, - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", + "type": "TRANSFORMED" + } + ] } - }, - "nativeDataType": "string", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users created on a single day", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null }, { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": [{ - "upstreamType": "FIELD_SET", - "upstreams": ["urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD),event_data)"], - "downstreamType": "FIELD", - "downstreams": ["urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_name)"], - "confidenceScore": 1.0 - }] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", - "documentSchemaType": "AVRO" - } - }, - "fields": [ - { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user created" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + ] } - }, - "nativeDataType": "varchar(100)", - "recursive": false }, { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who signed up" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} + "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata": { + "created": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "editableSchemaFieldInfo": [ + { + "fieldPath": "shipment_info", + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + } + } + ] } - }, - "nativeDataType": "boolean", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [{ - "name": "user id", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "table containing all the users deleted on a single day", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null }, { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", - "type": "TRANSFORMED" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ - { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", - "documentSchemaType": "AVRO" - } - }, - "fields": [ - { - "fieldPath": "user_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of the user who was deleted" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHdfsSchema", + "platform": "urn:li:dataPlatform:hdfs", + "version": 0, + "created": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", + "documentSchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "shipment_info", + "nullable": false, + "description": "Shipment info description", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipment_info.date", + "nullable": false, + "description": "Shipment info date description", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "Date", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipment_info.target", + "nullable": false, + "description": "Shipment info target description", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipment_info.destination", + "nullable": false, + "description": "Shipment info destination description", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipment_info.geo_info", + "nullable": false, + "description": "Shipment info geo_info description", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipment_info.geo_info.lat", + "nullable": false, + "description": "Shipment info geo_info lat", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "shipment_info.geo_info.lng", + "nullable": false, + "description": "Shipment info geo_info lng", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false, + "isPartOfKey": false + } + ] } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741908, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "aspects": [ { - "fieldPath": "timestamp", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Timestamp user was deleted at" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.common.Status": { + "removed": false } - }, - "nativeDataType": "long", - "recursive": false }, { - "fieldPath": "user_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Id of the user deleted" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } } - }, - "nativeDataType": "varchar(100)", - "recursive": false }, { - "fieldPath": "browser_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Cookie attached to identify the browser" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "type": "TRANSFORMED" + } + ] } - }, - "nativeDataType": "varchar(100)", - "recursive": false }, { - "fieldPath": "session_id", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Cookie attached to identify the session" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + ] } - }, - "nativeDataType": "varchar(100)", - "recursive": false }, { - "fieldPath": "deletion_reason", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Why the user chose to deactivate" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", + "documentSchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "field_foo", + "nullable": false, + "description": "Foo field description", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "field_bar", + "nullable": false, + "description": "Bar field description", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + } + ] } - }, - "nativeDataType": "varchar(100)", - "recursive": false - } - ], - "primaryKeys": ["user_name"], - "foreignKeysSpecs": null, - "foreignKeys": [{ - "name": "user session", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" - }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "name": "User Creations", - "description": "Constructs the fct_users_created from logging_events", - "type": "SQL", - "flowUrn": "urn:li:dataFlow:(airflow,dag_abc,PROD)" - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { - "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_456)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInfo": { - "name": "User Deletions", - "description": "Constructs the fct_users_deleted from logging_events", - "type": "SQL", - "flowUrn": "urn:li:dataFlow:(airflow,dag_abc,PROD)" - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { - "inputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)" - ], - "outputDatasets": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" - ], - "inputDatajobs": [ - "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)" - ] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { - "urn": "urn:li:dataFlow:(airflow,dag_abc,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { - "name": "Users", - "description": "Constructs the fct_users_deleted and fct_users_created tables", - "project": null - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(looker,baz1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "title": "Baz Chart 1", - "description": "Baz Chart 1", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": [ - "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)" - ], - "type": null, - "access": null, - "lastRefreshed": null - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [{ "tag": "urn:li:tag:Legacy" }] - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(looker,baz2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "title": "Baz Chart 2", - "description": "Baz Chart 2", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "chartUrl": null, - "inputs": { - "array": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)" - } - ] - }, - "type": null, - "access": null, - "lastRefreshed": null - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { - "urn": "urn:li:dashboard:(looker,baz)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpGroup:bfoo", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "title": "Baz Dashboard", - "description": "Baz Dashboard", - "customProperties": { - "prop1": "fakeprop", - "prop2": "pikachu" - }, - "charts": [ - "urn:li:chart:(looker,baz1)", - "urn:li:chart:(looker,baz2)" - ], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - }, - "deleted": null - }, - "dashboardUrl": null, - "access": null, - "lastRefreshed": null - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { - "urn": "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" }, { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER" + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + } } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:jdoe" - } - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { - "description": "A sample model for predicting some outcome.", - "date": null, - "tags": ["Sample"], - "version": null, - "type": "Naive Bayes classifier" - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.TrainingData": { - "trainingData": [ + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741910, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)", + "aspects": [ { - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,pageViewsHive,PROD)", - "motivation": "For science!", - "preProcessing": ["Aggregation"] - } - ] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.EvaluationData": { - "evaluationData": [ + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, { - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,pageViewsHive,PROD)" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.InstitutionalMemory": { - "elements": [ + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "encoding": "utf-8" + }, + "qualifiedName": "a.b.c", + "description": "table where each row represents a single log event", + "tags": [] + } + }, { - "url": "https://www.linkedin.com", - "description": "Sample doc", - "createStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - ] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.IntendedUse": { - "primaryUses": ["Sample Model", "Primary Use"], - "primaryUsers": ["ENTERPRISE"], - "outOfScopeUses": ["Production Deployment"] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.Metrics": { - "performanceMeasures": ["performanceMeasures"], - "decisionThreshold": ["decisionThreshold"] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.EthicalConsiderations": { - "data": ["data"], - "humanLife": ["humanLife"], - "mitigations": ["mitigations"], - "risksAndHarms": ["risksAndHarms"], - "useCases": ["useCases"] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.CaveatsAndRecommendations": { - "recommendations": "recommendations", - "idealDatasetCharacteristics": ["idealDatasetCharacteristics"] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Cost": { - "costType": "ORG_COST_TYPE", - "cost": { - "fieldDiscriminator": "costCode", - "costCode": "sampleCostCode" - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:Legacy", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "Legacy", - "description": "Indicates the dataset is no longer supported" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + }, { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { - "urn": "urn:li:tag:NeedsDocumentation", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.tag.TagProperties": { - "name": "NeedsDocumentation", - "description": "Indicates the data element needs documentation" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", + "documentSchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "event_name", + "nullable": false, + "description": "Name of your logging event", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "event_data", + "nullable": false, + "description": "Data of your event", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "timestamp", + "nullable": false, + "description": "TS the event was ingested", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "browser", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + } + ] + } } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - } - ] - } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741911, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:adlsGen1", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": "/", - "name": "adlsGen1", - "displayName": "Azure Data Lake (Gen 1)", - "type": "FILE_SYSTEM", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/adlslogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "encoding": "utf-8" + }, + "description": "table containing all the users created on a single day", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD),event_data)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", + "documentSchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "user_id", + "nullable": false, + "description": "Id of the user created", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_name", + "nullable": false, + "description": "Name of the user who signed up", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + } + ], + "foreignKeys": [ + { + "name": "user id", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741912, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:adlsGen2", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": "/", - "name": "adlsGen2", - "displayName": "Azure Data Lake (Gen 2)", - "type": "FILE_SYSTEM", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/adlslogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "encoding": "utf-8" + }, + "description": "table containing all the users deleted on a single day", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", + "documentSchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "user_name", + "nullable": false, + "description": "Name of the user who was deleted", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "timestamp", + "nullable": false, + "description": "Timestamp user was deleted at", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_id", + "nullable": false, + "description": "Id of the user deleted", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "browser_id", + "nullable": false, + "description": "Cookie attached to identify the browser", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "session_id", + "nullable": false, + "description": "Cookie attached to identify the session", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "deletion_reason", + "nullable": false, + "description": "Why the user chose to deactivate", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": false + } + ], + "primaryKeys": [ + "user_name" + ], + "foreignKeys": [ + { + "name": "user session", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + } + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741913, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:ambry", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "ambry", - "displayName": "Ambry", - "type": "OBJECT_STORE" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:datahub" + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": {}, + "name": "User Creations", + "description": "Constructs the fct_users_created from logging_events", + "type": { + "string": "SQL" + }, + "flowUrn": "urn:li:dataFlow:(airflow,dag_abc,PROD)" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741914, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:couchbase", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "couchbase", - "displayName": "Couchbase", - "type": "KEY_VALUE_STORE", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/couchbaselogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_456)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:datahub" + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": {}, + "name": "User Deletions", + "description": "Constructs the fct_users_deleted from logging_events", + "type": { + "string": "SQL" + }, + "flowUrn": "urn:li:dataFlow:(airflow,dag_abc,PROD)" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + ], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741915, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:external", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "external", - "displayName": "External Source", - "type": "OTHERS" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(airflow,dag_abc,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:datahub" + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": {}, + "name": "Users", + "description": "Constructs the fct_users_deleted and fct_users_created tables" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741915, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:hdfs", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": "/", - "name": "hdfs", - "displayName": "HDFS", - "type": "FILE_SYSTEM", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/hadooplogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,baz1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "title": "Baz Chart 1", + "description": "Baz Chart 1", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741916, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:hive", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "hive", - "displayName": "Hive", - "type": "FILE_SYSTEM", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/hivelogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(looker,baz2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "title": "Baz Chart 2", + "description": "Baz Chart 2", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)" + } + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741916, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:s3", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": "/", - "name": "s3", - "displayName": "AWS S3", - "type": "FILE_SYSTEM", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/s3.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(looker,baz)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpGroup:bfoo", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "prop1": "fakeprop", + "prop2": "pikachu" + }, + "title": "Baz Dashboard", + "description": "Baz Dashboard", + "charts": [ + "urn:li:chart:(looker,baz1)", + "urn:li:chart:(looker,baz2)" + ], + "datasets": [], + "dashboards": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:datahub" + } + } + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741916, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:kafka", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "kafka", - "displayName": "Kafka", - "type": "MESSAGE_BROKER", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/kafkalogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLModelSnapshot": { + "urn": "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:jdoe" + } + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties": { + "customProperties": {}, + "description": "A sample model for predicting some outcome.", + "type": "Naive Bayes classifier", + "tags": [ + "Sample" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.TrainingData": { + "trainingData": [ + { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,pageViewsHive,PROD)", + "motivation": "For science!", + "preProcessing": [ + "Aggregation" + ] + } + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.EvaluationData": { + "evaluationData": [ + { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,pageViewsHive,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.InstitutionalMemory": { + "elements": [ + { + "url": "https://www.linkedin.com", + "description": "Sample doc", + "createStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.IntendedUse": { + "primaryUses": [ + "Sample Model", + "Primary Use" + ], + "primaryUsers": [ + "ENTERPRISE" + ], + "outOfScopeUses": [ + "Production Deployment" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.Metrics": { + "performanceMeasures": [ + "performanceMeasures" + ], + "decisionThreshold": [ + "decisionThreshold" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.EthicalConsiderations": { + "data": [ + "data" + ], + "humanLife": [ + "humanLife" + ], + "mitigations": [ + "mitigations" + ], + "risksAndHarms": [ + "risksAndHarms" + ], + "useCases": [ + "useCases" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.CaveatsAndRecommendations": { + "recommendations": "recommendations", + "idealDatasetCharacteristics": [ + "idealDatasetCharacteristics" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Cost": { + "costType": "ORG_COST_TYPE", + "cost": { + "costCode": "sampleCostCode", + "fieldDiscriminator": "costCode" + } + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741917, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:kusto", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "kusto", - "displayName": "Kusto", - "type": "OLAP_DATASTORE", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/kustologo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:Legacy", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "Legacy", + "description": "Indicates the dataset is no longer supported" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741918, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:mongodb", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "mongodb", - "displayName": "MongoDB", - "type": "KEY_VALUE_STORE", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/mongodblogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.TagSnapshot": { + "urn": "urn:li:tag:NeedsDocumentation", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.tag.TagProperties": { + "name": "NeedsDocumentation", + "description": "Indicates the data element needs documentation" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741918, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:mysql", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "mysql", - "displayName": "MySQL", - "type": "RELATIONAL_DB", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/mysqllogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:adlsGen1", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "adlsGen1", + "displayName": "Azure Data Lake (Gen 1)", + "type": "FILE_SYSTEM", + "datasetNameDelimiter": "/", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/adlslogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741919, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:oracle", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "oracle", - "displayName": "Oracle", - "type": "RELATIONAL_DB", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/oraclelogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:adlsGen2", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "adlsGen2", + "displayName": "Azure Data Lake (Gen 2)", + "type": "FILE_SYSTEM", + "datasetNameDelimiter": "/", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/adlslogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741919, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:pinot", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "pinot", - "displayName": "Pinot", - "type": "OLAP_DATASTORE", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/pinotlogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:ambry", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "ambry", + "displayName": "Ambry", + "type": "OBJECT_STORE", + "datasetNameDelimiter": "." + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741919, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:postgres", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "postgres", - "displayName": "PostgreSQL", - "type": "RELATIONAL_DB", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/postgreslogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:couchbase", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "couchbase", + "displayName": "Couchbase", + "type": "KEY_VALUE_STORE", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/couchbaselogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741919, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:presto", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "presto", - "displayName": "Presto", - "type": "QUERY_ENGINE", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/prestologo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:external", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "external", + "displayName": "External Source", + "type": "OTHERS", + "datasetNameDelimiter": "." + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741919, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:teradata", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "teradata", - "displayName": "Teradata", - "type": "RELATIONAL_DB", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/teradatalogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:hdfs", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "hdfs", + "displayName": "HDFS", + "type": "FILE_SYSTEM", + "datasetNameDelimiter": "/", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/hadooplogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741920, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:voldemort", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "voldemort", - "displayName": "Voldemort", - "type": "KEY_VALUE_STORE" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:hive", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "hive", + "displayName": "Hive", + "type": "FILE_SYSTEM", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/hivelogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741920, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:snowflake", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "snowflake", - "displayName": "Snowflake", - "type": "RELATIONAL_DB", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/snowflakelogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:s3", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "s3", + "displayName": "AWS S3", + "type": "FILE_SYSTEM", + "datasetNameDelimiter": "/", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/s3.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741920, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:redshift", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "redshift", - "displayName": "Redshift", - "type": "RELATIONAL_DB", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/redshiftlogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:kafka", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "kafka", + "displayName": "Kafka", + "type": "MESSAGE_BROKER", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/kafkalogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741920, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:mssql", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "mssql", - "displayName": "SQL Server", - "type": "RELATIONAL_DB", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/mssqllogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:kusto", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "kusto", + "displayName": "Kusto", + "type": "OLAP_DATASTORE", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/kustologo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741920, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:bigquery", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "bigquery", - "displayName": "BigQuery", - "type": "RELATIONAL_DB", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/bigquerylogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:mongodb", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "mongodb", + "displayName": "MongoDB", + "type": "KEY_VALUE_STORE", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/mongodblogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741921, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:druid", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "druid", - "displayName": "Druid", - "type": "OLAP_DATASTORE", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/druidlogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:mysql", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "mysql", + "displayName": "MySQL", + "type": "RELATIONAL_DB", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/mysqllogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741921, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:looker", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "looker", - "displayName": "Looker", - "type": "OTHERS", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/lookerlogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:oracle", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "oracle", + "displayName": "Oracle", + "type": "RELATIONAL_DB", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/oraclelogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741921, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:feast", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "feast", - "displayName": "Feast", - "type": "OTHERS", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/feastlogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:pinot", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "pinot", + "displayName": "Pinot", + "type": "OLAP_DATASTORE", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/pinotlogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741921, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:sagemaker", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "sagemaker", - "displayName": "SageMaker", - "type": "OTHERS", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/sagemakerlogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:postgres", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "postgres", + "displayName": "PostgreSQL", + "type": "RELATIONAL_DB", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/postgreslogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741921, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { - "urn": "urn:li:dataPlatform:glue", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { - "datasetNameDelimiter": ".", - "name": "glue", - "displayName": "Glue", - "type": "OTHERS", - "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/gluelogo.png" - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:presto", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "presto", + "displayName": "Presto", + "type": "QUERY_ENGINE", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/prestologo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741921, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": "Dummy entity 1", - "dataType": "TEXT", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:teradata", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "teradata", + "displayName": "Teradata", + "type": "RELATIONAL_DB", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/teradatalogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741922, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": "Dummy entity 2", - "dataType": "ORDINAL", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:voldemort", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "voldemort", + "displayName": "Voldemort", + "type": "KEY_VALUE_STORE", + "datasetNameDelimiter": "." + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741922, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "SEQUENCE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:snowflake", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "snowflake", + "displayName": "Snowflake", + "type": "RELATIONAL_DB", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/snowflakelogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741922, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "BINARY", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:redshift", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "redshift", + "displayName": "Redshift", + "type": "RELATIONAL_DB", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/redshiftlogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741922, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BYTES_LIST_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "SEQUENCE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:mssql", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "mssql", + "displayName": "SQL Server", + "type": "RELATIONAL_DB", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/mssqllogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741922, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BYTES_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "BYTE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:bigquery", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "bigquery", + "displayName": "BigQuery", + "type": "RELATIONAL_DB", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/bigquerylogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741923, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_DOUBLE_LIST_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "SEQUENCE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:druid", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "druid", + "displayName": "Druid", + "type": "OLAP_DATASTORE", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/druidlogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741923, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_DOUBLE_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "CONTINUOUS", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:looker", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "looker", + "displayName": "Looker", + "type": "OTHERS", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/lookerlogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741923, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_FLOAT_LIST_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "SEQUENCE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:feast", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "feast", + "displayName": "Feast", + "type": "OTHERS", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/feastlogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741923, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_FLOAT_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "CONTINUOUS", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:sagemaker", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "sagemaker", + "displayName": "SageMaker", + "type": "OTHERS", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/sagemakerlogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741923, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT32_LIST_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "SEQUENCE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": { + "urn": "urn:li:dataPlatform:glue", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": { + "name": "glue", + "displayName": "Glue", + "type": "OTHERS", + "datasetNameDelimiter": ".", + "logoUrl": "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/gluelogo.png" + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741923, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT32_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "ORDINAL", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "customProperties": {}, + "description": "Dummy entity 1", + "dataType": "TEXT", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741924, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT64_LIST_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "SEQUENCE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "customProperties": {}, + "description": "Dummy entity 2", + "dataType": "ORDINAL", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741924, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT64_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "ORDINAL", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "SEQUENCE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741924, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_STRING_LIST_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "SEQUENCE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "BINARY", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741924, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_STRING_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "TEXT", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BYTES_LIST_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "SEQUENCE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741925, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,test_feature_table_all_feature_dtypes)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/feast/test_feature_table_all_feature_dtypes"] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": {}, - "description": null, - "mlFeatures": [ - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BYTES_LIST_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BYTES_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_DOUBLE_LIST_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_DOUBLE_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_FLOAT_LIST_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_FLOAT_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT32_LIST_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT32_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT64_LIST_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT64_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_STRING_LIST_feature)", - "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_STRING_feature)" - ], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_1)", - "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_2)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BYTES_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "BYTE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741925, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(test_feature_table_no_labels,dummy_entity_2)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": "Dummy entity 2", - "dataType": "ORDINAL", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_DOUBLE_LIST_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "SEQUENCE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741925, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_no_labels,test_BYTES_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "BYTE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_DOUBLE_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "CONTINUOUS", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741925, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,test_feature_table_no_labels)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/feast/test_feature_table_no_labels"] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": {}, - "description": null, - "mlFeatures": [ - "urn:li:mlFeature:(test_feature_table_no_labels,test_BYTES_feature)" - ], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(test_feature_table_no_labels,dummy_entity_2)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_FLOAT_LIST_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "SEQUENCE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741926, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(test_feature_table_single_feature,dummy_entity_1)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": "Dummy entity 1", - "dataType": "TEXT", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_FLOAT_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "CONTINUOUS", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741926, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(test_feature_table_single_feature,test_BYTES_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "BYTE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT32_LIST_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "SEQUENCE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741926, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,test_feature_table_single_feature)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/feast/test_feature_table_single_feature"] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": {}, - "description": null, - "mlFeatures": [ - "urn:li:mlFeature:(test_feature_table_single_feature,test_BYTES_feature)" - ], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(test_feature_table_single_feature,dummy_entity_1)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT32_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "ORDINAL", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741926, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(user_features,user_name)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": "User's user_name", - "dataType": "TEXT", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT64_LIST_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "SEQUENCE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741927, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(user_features,user_id)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": "User's internal ID", - "dataType": "ORDINAL", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT64_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "ORDINAL", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741927, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(user_features,regions)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "SEQUENCE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_STRING_LIST_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "SEQUENCE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741927, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(user_features,is_premium_user)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "BINARY", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_STRING_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "TEXT", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741927, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(user_features,number_of_visits)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "ORDINAL", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,test_feature_table_all_feature_dtypes)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/feast/test_feature_table_all_feature_dtypes" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": {}, + "mlFeatures": [ + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BYTES_LIST_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BYTES_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_DOUBLE_LIST_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_DOUBLE_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_FLOAT_LIST_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_FLOAT_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT32_LIST_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT32_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT64_LIST_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_INT64_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_STRING_LIST_feature)", + "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_STRING_feature)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_1)", + "urn:li:mlPrimaryKey:(test_feature_table_all_feature_dtypes,dummy_entity_2)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741928, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_features)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/feast/user_features"] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": {}, - "description": null, - "mlFeatures": [ - "urn:li:mlFeature:(user_features,regions)", - "urn:li:mlFeature:(user_features,is_premium_user)", - "urn:li:mlFeature:(user_features,number_of_visits)" - ], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(user_features,user_name)", - "urn:li:mlPrimaryKey:(user_features,user_id)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(test_feature_table_no_labels,dummy_entity_2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "customProperties": {}, + "description": "Dummy entity 2", + "dataType": "ORDINAL", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741928, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { - "urn": "urn:li:mlPrimaryKey:(user_analytics,user_name)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { - "description": "User's username", - "dataType": "TEXT", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_no_labels,test_BYTES_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "BYTE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741928, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { - "urn": "urn:li:mlFeature:(user_analytics,date_joined)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { - "description": null, - "dataType": "BYTE", - "version": null, - "sources": [ - "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,test_feature_table_no_labels)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/feast/test_feature_table_no_labels" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": {}, + "mlFeatures": [ + "urn:li:mlFeature:(test_feature_table_no_labels,test_BYTES_feature)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(test_feature_table_no_labels,dummy_entity_2)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741928, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { - "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_analytics)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": ["/feast/user_analytics"] - } - }, - { - "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { - "customProperties": {}, - "description": null, - "mlFeatures": ["urn:li:mlFeature:(user_analytics,date_joined)"], - "mlPrimaryKeys": [ - "urn:li:mlPrimaryKey:(user_analytics,user_name)" - ] - } - } - ] - } + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(test_feature_table_single_feature,dummy_entity_1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "customProperties": {}, + "description": "Dummy entity 1", + "dataType": "TEXT", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { + "systemMetadata": { + "lastObserved": 1758537741929, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(test_feature_table_single_feature,test_BYTES_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "BYTE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:file,feast.*,PROD)" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741929, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,test_feature_table_single_feature)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/feast/test_feature_table_single_feature" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": {}, + "mlFeatures": [ + "urn:li:mlFeature:(test_feature_table_single_feature,test_BYTES_feature)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(test_feature_table_single_feature,dummy_entity_1)" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741929, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(user_features,user_name)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "customProperties": {}, + "description": "User's user_name", + "dataType": "TEXT", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741929, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(user_features,user_id)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "customProperties": {}, + "description": "User's internal ID", + "dataType": "ORDINAL", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741929, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(user_features,regions)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "SEQUENCE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741930, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(user_features,is_premium_user)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "BINARY", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741930, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(user_features,number_of_visits)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "ORDINAL", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741930, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_features)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/feast/user_features" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": {}, + "mlFeatures": [ + "urn:li:mlFeature:(user_features,regions)", + "urn:li:mlFeature:(user_features,is_premium_user)", + "urn:li:mlFeature:(user_features,number_of_visits)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(user_features,user_name)", + "urn:li:mlPrimaryKey:(user_features,user_id)" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741930, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLPrimaryKeySnapshot": { + "urn": "urn:li:mlPrimaryKey:(user_analytics,user_name)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "customProperties": {}, + "description": "User's username", + "dataType": "TEXT", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741931, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureSnapshot": { + "urn": "urn:li:mlFeature:(user_analytics,date_joined)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, + "dataType": "BYTE", + "sources": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741931, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.MLFeatureTableSnapshot": { + "urn": "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,user_analytics)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/feast/user_analytics" + ] + } + }, + { + "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { + "customProperties": {}, + "mlFeatures": [ + "urn:li:mlFeature:(user_analytics,date_joined)" + ], + "mlPrimaryKeys": [ + "urn:li:mlPrimaryKey:(user_analytics,user_name)" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741931, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { "urn": "urn:li:glossaryTerm:CustomerAccount", "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { + "customProperties": { + "FQDN": "CustomerAccount" + }, + "definition": "account that represents an identified, named collection of balances and cumulative totals used to summarize customer transaction-related activity over a designated period of time", + "parentNode": "urn:li:glossaryNode:ClientsAndAccounts", + "termSource": "EXTERNAL", + "sourceRef": "FIBO", + "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/ProductsAndServices/ClientsAndAccounts/CustomerAccount" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } } - }, - { - "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { - "definition": "account that represents an identified, named collection of balances and cumulative totals used to summarize customer transaction-related activity over a designated period of time", - "parentNode": "urn:li:glossaryNode:ClientsAndAccounts", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/ProductsAndServices/ClientsAndAccounts/CustomerAccount", - "customProperties": { - "FQDN": "CustomerAccount" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [{ - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - }], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe" - } - } - } ] - } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { - "urn": "urn:li:glossaryTerm:SavingAccount", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { - "definition": "a product provided to consumers and businesses by a bank or similar depository institution such as a checking account, savings account, certificate of deposit, debit or pre-paid card, or credit card", - "parentNode": "urn:li:glossaryNode:ClientsAndAccounts", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct", - "customProperties": { - "FQDN": "SavingAccount" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [{ - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - }], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe" - } - } - } - ] } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { + }, + "systemMetadata": { + "lastObserved": 1758537741931, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { - "urn": "urn:li:glossaryTerm:AccountBalance", - "aspects": [{ - "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { - "definition": "amount of money available or owed", - "parentNode": "urn:li:glossaryNode:ClientsAndAccounts", - "sourceRef": "FIBO", - "termSource": "EXTERNAL", - "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/ProductsAndServices/ClientsAndAccounts/Balance", - "customProperties": { - "FQDN": "AccountBalance" - } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [{ - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - }], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe" + "urn": "urn:li:glossaryTerm:SavingAccount", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { + "customProperties": { + "FQDN": "SavingAccount" + }, + "definition": "a product provided to consumers and businesses by a bank or similar depository institution such as a checking account, savings account, certificate of deposit, debit or pre-paid card, or credit card", + "parentNode": "urn:li:glossaryNode:ClientsAndAccounts", + "termSource": "EXTERNAL", + "sourceRef": "FIBO", + "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/FunctionalEntities/FinancialServicesEntities/BankingProduct" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } } - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] + ] } - }, - "proposedDelta": null - }, - { - "auditHeader": null, - "proposedSnapshot": { + }, + "systemMetadata": { + "lastObserved": 1758537741932, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryTermSnapshot": { + "urn": "urn:li:glossaryTerm:AccountBalance", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo": { + "customProperties": { + "FQDN": "AccountBalance" + }, + "definition": "amount of money available or owed", + "parentNode": "urn:li:glossaryNode:ClientsAndAccounts", + "termSource": "EXTERNAL", + "sourceRef": "FIBO", + "sourceUrl": "https://spec.edmcouncil.org/fibo/ontology/FBC/ProductsAndServices/ClientsAndAccounts/Balance" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741932, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.GlossaryNodeSnapshot": { - "urn": "urn:li:glossaryNode:ClientsAndAccounts", - "aspects": [{ - "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { - "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [{ - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER" - }], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe" + "urn": "urn:li:glossaryNode:ClientsAndAccounts", + "aspects": [ + { + "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": {}, + "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities" + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } } - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] + ] } - }, - "proposedDelta": null }, - { - "auditHeader": null, - "proposedSnapshot": { + "systemMetadata": { + "lastObserved": 1758537741932, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,project/root/events/logging_events_bckp,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "description": "s3 backup of logging events", - "uri": null, - "tags": [], - "customProperties": { - "encoding": "utf-8" + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,project/root/events/logging_events_bckp,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "encoding": "utf-8" + }, + "description": "s3 backup of logging events", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)", + "type": "COPY" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "SampleHiveSchema", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 1674291843000, + "actor": "urn:li:corpuser:jdoe" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", + "documentSchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "event_name", + "nullable": false, + "description": "Name of your logging event", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "event_data", + "nullable": false, + "description": "Data of your event", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "timestamp", + "nullable": false, + "description": "TS the event was ingested", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "browser", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + } + ] + } } - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe", - "type": "DATAOWNER", - "source": null - }, - { - "owner": "urn:li:corpuser:datahub", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)", - "type": "COPY" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "SampleHiveSchema", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "lastModified": { - "time": 1674291843000, - "actor": "urn:li:corpuser:jdoe", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.KafkaSchema": { - "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}", - "documentSchemaType": "AVRO" - } - }, - "fields": [ - { - "fieldPath": "event_name", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Name of your logging event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "varchar(100)", - "recursive": false - }, - { - "fieldPath": "event_data", - "jsonPath": null, - "nullable": false, - "description": { - "string": "Data of your event" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - }, - { - "fieldPath": "timestamp", - "jsonPath": null, - "nullable": false, - "description": { - "string": "TS the event was ingested" - }, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false - }, - { - "fieldPath": "browser", - "jsonPath": null, - "nullable": false, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null - } - } - ] + ] } - }, - "proposedDelta": null }, - { - "auditHeader":null, - "entityType":"dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "changeType":"UPSERT", - "aspectName":"datasetProfile", - "aspect":{ - "value":"{\"timestampMillis\": 1735823280000, \"rowCount\": 4500, \"columnCount\": 2, \"sizeInBytes\": 842000200000, \"fieldProfiles\": [{\"fieldPath\": \"field_foo\", \"uniqueCount\": 2, \"uniqueProportion\": 0.00044, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"true\", \"false\"]}, {\"fieldPath\": \"field_bar\", \"uniqueCount\": 2, \"uniqueProportion\": 0.00044, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"false\"]}]}", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader":null, - "entityType":"dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "changeType":"UPSERT", - "aspectName":"datasetProfile", - "aspect":{ - "value":"{\"timestampMillis\": 1723488954865, \"rowCount\": 3500, \"columnCount\": 2, \"fieldProfiles\": [{\"fieldPath\": \"field_foo\", \"uniqueCount\": 2, \"uniqueProportion\": 0.00057, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"true\", \"false\"]}, {\"fieldPath\": \"field_bar\", \"uniqueCount\": 2, \"uniqueProportion\": 0.00057, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"true\"]}]}", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader":null, - "entityType":"dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "changeType":"UPSERT", - "aspectName":"operation", - "aspect":{ - "value":"{\"timestampMillis\": 1711138093000, \"operationType\": \"INSERT\", \"lastUpdatedTimestamp\": 1629097200001 }", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader":null, - "entityType":"dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "changeType":"UPSERT", - "aspectName":"operation", - "aspect":{ - "value":"{\"timestampMillis\": 1629107200000, \"operationType\": \"UPDATE\", \"lastUpdatedTimestamp\": 1625097200000 }", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741933, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1735823280000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 4500, + "columnCount": 2, + "fieldProfiles": [ + { + "fieldPath": "field_foo", + "uniqueCount": 2, + "uniqueProportion": 0.00044, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "true", + "false" + ] + }, + { + "fieldPath": "field_bar", + "uniqueCount": 2, + "uniqueProportion": 0.00044, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "false" + ] + } + ], + "sizeInBytes": 842000200000 + } + }, + "systemMetadata": { + "lastObserved": 1758537741933, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1723488954865, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 3500, + "columnCount": 2, + "fieldProfiles": [ + { + "fieldPath": "field_foo", + "uniqueCount": 2, + "uniqueProportion": 0.00057, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "true", + "false" + ] + }, + { + "fieldPath": "field_bar", + "uniqueCount": 2, + "uniqueProportion": 0.00057, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "true" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741934, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1711138093000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "INSERT", + "lastUpdatedTimestamp": 1629097200001 + } + }, + "systemMetadata": { + "lastObserved": 1758537741934, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1629107200000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1625097200000 + } + }, + "systemMetadata": { + "lastObserved": 1758537741935, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { - "value": "{\"timestampMillis\": 1674291843000, \"eventGranularity\": {\"unit\": \"DAY\", \"multiple\": 1}, \"uniqueUserCount\": 1, \"totalSqlQueries\": 2, \"topSqlQueries\": [\"select * from `SampleHiveDataset`\", \"select field_foo, field_bar from `SampleHiveDataset`\"], \"userCounts\": [{\"user\": \"urn:li:corpuser:unknown\", \"count\": 2, \"userEmail\": \"test_user@datahubproject.io\"}], \"fieldCounts\": [{\"fieldPath\": \"field_foo\", \"count\": 45}, {\"fieldPath\": \"field_bar\", \"count\": 10}]}", - "contentType": "application/json" + "json": { + "timestampMillis": 1674291843000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "uniqueUserCount": 1, + "totalSqlQueries": 2, + "topSqlQueries": [ + "select * from `SampleHiveDataset`", + "select field_foo, field_bar from `SampleHiveDataset`" + ], + "userCounts": [ + { + "user": "urn:li:corpuser:unknown", + "count": 2, + "userEmail": "test_user@datahubproject.io" + } + ], + "fieldCounts": [ + { + "fieldPath": "field_foo", + "count": 45 + }, + { + "fieldPath": "field_bar", + "count": 10 + } + ] + } }, "systemMetadata": { - "lastObserved": 1674291843000, - "runId": "test-bigquery-usage", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1674291843000, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" } - }, - { - "auditHeader":null, - "entityType":"container", +}, +{ + "entityType": "container", "entityUrn": "urn:li:container:DATABASE", - "changeType":"UPSERT", - "aspectName":"containerProperties", - "aspect":{ - "value":"{\"name\": \"datahub_db\", \"description\": \"The Root DataHub DB\" }", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader":null, - "entityType":"container", - "entityUrn": "urn:li:container:DATABASE", - "changeType":"UPSERT", - "aspectName":"subTypes", - "aspect":{ - "value":"{\"typeNames\": [ \"Database\" ] }", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader":null, - "entityType":"container", - "entityUrn": "urn:li:container:DATABASE", - "changeType":"UPSERT", - "aspectName":"dataPlatformInstance", - "aspect":{ - "value":"{\"platform\": \"urn:li:dataPlatform:hive\" }", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader":null, - "entityType":"container", - "entityUrn": "urn:li:container:SCHEMA", - "changeType":"UPSERT", - "aspectName":"containerProperties", - "aspect":{ - "value":"{\"name\": \"datahub_schema\", \"description\": \"The Root DataHub Schema\" }", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader":null, - "entityType":"container", - "entityUrn": "urn:li:container:SCHEMA", - "changeType":"UPSERT", - "aspectName":"dataPlatformInstance", - "aspect":{ - "value":"{\"platform\": \"urn:li:dataPlatform:hive\" }", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader":null, - "entityType":"container", - "entityUrn": "urn:li:container:SCHEMA", - "changeType":"UPSERT", - "aspectName":"subTypes", - "aspect":{ - "value":"{\"typeNames\": [ \"Schema\" ] }", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader":null, - "entityType":"dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", - "changeType":"UPSERT", - "aspectName":"container", - "aspect":{ - "value":"{\"container\": \"urn:li:container:SCHEMA\" }", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader":null, - "entityType":"container", - "entityUrn": "urn:li:container:SCHEMA", - "changeType":"UPSERT", - "aspectName":"container", - "aspect":{ - "value":"{\"container\": \"urn:li:container:DATABASE\" }", - "contentType":"application/json" - }, - "systemMetadata":null - }, - { - "auditHeader": null, - "entityType": "assertion", - "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, "changeType": "UPSERT", - "aspectName": "assertionInfo", + "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"expectation_suite_name\": \"test_suite\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD),field_foo)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"true\", \"type\": \"UNKNOWN\"} }, \"nativeType\": \"expect_column_values_to_equal\", \"nativeParameters\": {\"column\": \"field_foo\", \"value\": \"true\"}}}", - "contentType": "application/json" + "json": { + "customProperties": {}, + "name": "datahub_db", + "description": "The Root DataHub DB" + } }, - "systemMetadata": null - }, - { - "auditHeader": null, - "entityType": "assertion", - "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, + "systemMetadata": { + "lastObserved": 1758537741935, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:DATABASE", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741936, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:DATABASE", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:great-expectations\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:hive" + } }, - "systemMetadata": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741936, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:SCHEMA", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "datahub_schema", + "description": "The Root DataHub Schema" + } + }, + "systemMetadata": { + "lastObserved": 1758537741936, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:SCHEMA", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:hive" + } + }, + "systemMetadata": { + "lastObserved": 1758537741936, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:SCHEMA", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1758537741936, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:SCHEMA" + } + }, + "systemMetadata": { + "lastObserved": 1758537741936, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:SCHEMA", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:DATABASE" + } + }, + "systemMetadata": { + "lastObserved": 1758537741937, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "expectation_suite_name": "test_suite" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD),field_foo)" + ], + "aggregation": "IDENTITY", + "operator": "EQUAL_TO", + "parameters": { + "value": { + "value": "true", + "type": "UNKNOWN" + } + }, + "nativeType": "expect_column_values_to_equal", + "nativeParameters": { + "column": "field_foo", + "value": "true" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1758537741937, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:great-expectations" + } + }, + "systemMetadata": { + "lastObserved": 1758537741937, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "entityType": "assertion", "entityUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { - "value": "{\"timestampMillis\": 1730554659000, \"partitionSpec\": {\"type\": \"PARTITION\", \"partition\": \"{\\\"category\\\": \\\"catA\\\"}\"}, \"runId\": \"2021-12-28T12:00:00Z\", \"assertionUrn\": \"urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)\", \"batchSpec\": {\"customProperties\": {\"data_asset_name\": \"data__foo1__asset\", \"datasource_name\": \"my_hive_datasource\"}, \"nativeBatchId\": \"c8f12129f2e57412eee5fb8656154d05\", \"limit\": 10}, \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", - "contentType": "application/json" + "json": { + "timestampMillis": 1730554659000, + "runId": "2021-12-28T12:00:00Z", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "batchSpec": { + "customProperties": { + "data_asset_name": "data__foo1__asset", + "datasource_name": "my_hive_datasource" + }, + "nativeBatchId": "c8f12129f2e57412eee5fb8656154d05", + "limit": 10 + }, + "assertionUrn": "urn:li:assertion:358c683782c93c2fc2bd4bdd4fdb0153", + "partitionSpec": { + "partition": "{\"category\": \"catA\"}", + "type": "PARTITION" + } + } }, - "systemMetadata": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741937, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "entityType": "query", "entityUrn": "urn:li:query:test-query", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { - "value": "{\"name\":\"TestQuery\",\"description\":\"TestDescription\",\"source\":\"MANUAL\",\"statement\":{\"language\":\"SQL\",\"value\":\"SELECT * FROM db.SampleHiveDataset WHERE field_foo = 'foo_value'\"},\"created\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0},\"lastModified\":{\"actor\":\"urn:li:corpuser:test\",\"time\":0}}", - "contentType": "application/json" + "json": { + "customProperties": {}, + "statement": { + "value": "SELECT * FROM db.SampleHiveDataset WHERE field_foo = 'foo_value'", + "language": "SQL" + }, + "source": "MANUAL", + "name": "TestQuery", + "description": "TestDescription", + "created": { + "time": 0, + "actor": "urn:li:corpuser:test" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:test" + } + } }, - "systemMetadata": null - }, - { - "auditHeader": null, + "systemMetadata": { + "lastObserved": 1758537741938, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ "entityType": "query", "entityUrn": "urn:li:query:test-query", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { - "value": "{\"subjects\":[{\"entity\":\"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)\"}]}", - "contentType": "application/json" + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" + } + ] + } }, - "systemMetadata": null - }, - { - "entityType": "post", - "entityUrn": "urn:li:post:f3a68539-f7e4-4c41-a4fd-9e57c085d8dd", - "changeType": "UPSERT", - "aspectName": "postInfo", - "aspect": { - "json": { - "type": "HOME_PAGE_ANNOUNCEMENT", - "content": { - "title": "Join DataHub Slack", - "type": "LINK", - "link": "https://datahub.com/slack?utm_source=quickstart&utm_medium=annoucement&utm_campaign=quickstart_annoucement", - "media": { - "type": "IMAGE", - "location": "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-logo-color-mark.svg" - } - }, - "created": 1712547125049, - "lastModified": 1712547125049 - } - }, - "systemMetadata": { - "lastObserved": 1712548844816, - "runId": "datahub-2024_04_08-13_00_44", - "lastRunId": "no-run-id-provided" - } - }, - { - "entityType": "post", - "entityUrn": "urn:li:post:e9c7e4a5-b7b0-4390-bd94-9a0fe8acd6bf", - "changeType": "UPSERT", - "aspectName": "postInfo", - "aspect": { - "json": { - "type": "HOME_PAGE_ANNOUNCEMENT", - "content": { - "title": "View Docs", - "type": "LINK", - "link": "https://docs.datahub.com/docs", - "media": { - "type": "IMAGE", - "location": "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-logo-color-mark.svg" - } - }, - "created": 1712547164761, - "lastModified": 1712547164761 - } - }, - "systemMetadata": { - "lastObserved": 1712548844816, - "runId": "datahub-2024_04_08-13_00_44", - "lastRunId": "no-run-id-provided" - } - }, - { - "entityType": "post", - "entityUrn": "urn:li:post:326e42b3-2965-47d1-9660-50e513af7d6e", - "changeType": "UPSERT", - "aspectName": "postInfo", - "aspect": { - "json": { - "type": "HOME_PAGE_ANNOUNCEMENT", - "content": { - "title": "Upcoming Events", - "type": "LINK", - "link": "https://datahub.com/events/", - "media": { - "type": "IMAGE", - "location": "https://docs.datahub.com/img/datahub-logo-color-mark.svg" - } - }, - "created": 1712547268882, - "lastModified": 1712547268882 - } - }, - "systemMetadata": { - "lastObserved": 1712548844817, - "runId": "datahub-2024_04_08-13_00_44", - "lastRunId": "no-run-id-provided" - } - } -] + "systemMetadata": { + "lastObserved": 1758537741938, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "post", + "entityUrn": "urn:li:post:f3a68539-f7e4-4c41-a4fd-9e57c085d8dd", + "changeType": "UPSERT", + "aspectName": "postInfo", + "aspect": { + "json": { + "type": "HOME_PAGE_ANNOUNCEMENT", + "content": { + "title": "Join DataHub Slack", + "type": "LINK", + "link": "https://datahub.com/slack?utm_source=quickstart&utm_medium=annoucement&utm_campaign=quickstart_annoucement", + "media": { + "type": "IMAGE", + "location": "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-logo-color-mark.svg" + } + }, + "created": 1712547125049, + "lastModified": 1712547125049 + } + }, + "systemMetadata": { + "lastObserved": 1712548844816, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "post", + "entityUrn": "urn:li:post:e9c7e4a5-b7b0-4390-bd94-9a0fe8acd6bf", + "changeType": "UPSERT", + "aspectName": "postInfo", + "aspect": { + "json": { + "type": "HOME_PAGE_ANNOUNCEMENT", + "content": { + "title": "View Docs", + "type": "LINK", + "link": "https://docs.datahub.com/docs", + "media": { + "type": "IMAGE", + "location": "https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-logo-color-mark.svg" + } + }, + "created": 1712547164761, + "lastModified": 1712547164761 + } + }, + "systemMetadata": { + "lastObserved": 1712548844816, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "post", + "entityUrn": "urn:li:post:326e42b3-2965-47d1-9660-50e513af7d6e", + "changeType": "UPSERT", + "aspectName": "postInfo", + "aspect": { + "json": { + "type": "HOME_PAGE_ANNOUNCEMENT", + "content": { + "title": "Upcoming Events", + "type": "LINK", + "link": "https://datahub.com/events/", + "media": { + "type": "IMAGE", + "location": "https://docs.datahub.com/img/datahub-logo-color-mark.svg" + } + }, + "created": 1712547268882, + "lastModified": 1712547268882 + } + }, + "systemMetadata": { + "lastObserved": 1712548844817, + "runId": "demo-data-2025_09_22-16_12_20-modc1r", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file