datahub/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json

785 lines
28 KiB
JSON
Raw Permalink Normal View History

[
{
"entityType": "container",
"entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "postgres",
"env": "PROD",
"database": "postgrestest"
},
"name": "postgrestest",
"env": "PROD"
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:postgres"
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Database"
]
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": []
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:a6097853edba03be190d99ece4b307ff"
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "postgres",
"env": "PROD",
"database": "postgrestest",
"schema": "public"
},
"name": "public",
"env": "PROD"
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:postgres"
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Schema"
]
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
"urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
}
]
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:51904fc8cd5cc729bc630decff284525"
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.common.Status": {
"removed": false
}
},
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {},
"name": "metadata_aspect_v2",
"tags": []
}
},
{
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "postgrestest.public.metadata_aspect_v2",
"platform": "urn:li:dataPlatform:postgres",
"version": 0,
"created": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"hash": "",
"platformSchema": {
"com.linkedin.pegasus2avro.schema.MySqlDDL": {
"tableSchema": ""
}
},
"fields": [
{
"fieldPath": "urn",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "VARCHAR(500)",
"recursive": false,
"glossaryTerms": {
"terms": [
{
"urn": "urn:li:glossaryTerm:URN"
}
],
"auditStamp": {
"time": 1646575200000,
"actor": "urn:li:corpuser:datahub"
}
},
"isPartOfKey": true
},
{
"fieldPath": "aspect",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "VARCHAR(200)",
"recursive": false,
"isPartOfKey": true
},
{
"fieldPath": "version",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "BIGINT",
"recursive": false,
"isPartOfKey": true
},
{
"fieldPath": "metadata",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "TEXT",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "systemmetadata",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "TEXT",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "createdon",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.TimeType": {}
}
},
"nativeDataType": "TIMESTAMP WITHOUT TIME ZONE",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "createdby",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "VARCHAR(255)",
"recursive": false,
"glossaryTerms": {
"terms": [
{
"urn": "urn:li:glossaryTerm:URN"
}
],
"auditStamp": {
"time": 1646575200000,
"actor": "urn:li:corpuser:datahub"
}
},
"isPartOfKey": false
},
{
"fieldPath": "createdfor",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "VARCHAR(255)",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "metadata_json",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.RecordType": {}
}
},
"nativeDataType": "JSON",
"recursive": false,
"isPartOfKey": false
}
]
}
}
]
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Table"
]
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
"urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
},
{
"id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
"urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
}
]
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:51904fc8cd5cc729bc630decff284525"
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.common.Status": {
"removed": false
}
},
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"is_view": "True",
"view_definition": " SELECT metadata_aspect_v2.urn,\n metadata_aspect_v2.aspect\n FROM metadata_aspect_v2\n WHERE (metadata_aspect_v2.version = 0);"
},
"name": "metadata_aspect_view",
"tags": []
}
},
{
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "postgrestest.public.metadata_aspect_view",
"platform": "urn:li:dataPlatform:postgres",
"version": 0,
"created": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"hash": "",
"platformSchema": {
"com.linkedin.pegasus2avro.schema.MySqlDDL": {
"tableSchema": ""
}
},
"fields": [
{
"fieldPath": "urn",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "VARCHAR(500)",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "aspect",
"nullable": true,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.StringType": {}
}
},
"nativeDataType": "VARCHAR(200)",
"recursive": false,
"isPartOfKey": false
}
]
}
}
]
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"View"
]
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
"changeType": "UPSERT",
"aspectName": "viewProperties",
"aspect": {
"json": {
"materialized": false,
"viewLogic": " SELECT metadata_aspect_v2.urn,\n metadata_aspect_v2.aspect\n FROM metadata_aspect_v2\n WHERE (metadata_aspect_v2.version = 0);",
"viewLanguage": "SQL"
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
"urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
},
{
"id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
"urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
}
]
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
"json": {
"timestampMillis": 1646575200000,
"partitionSpec": {
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
"rowCount": 2,
"columnCount": 9,
"fieldProfiles": [],
"sizeInBytes": 16384
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
"changeType": "UPSERT",
"aspectName": "upstreamLineage",
"aspect": {
"json": {
"upstreams": [
{
"auditStamp": {
"time": 1646575200000,
"actor": "urn:li:corpuser:_ingestion"
},
"created": {
"time": 0,
"actor": "urn:li:corpuser:_ingestion"
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
"type": "VIEW",
"query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29"
}
],
"fineGrainedLineages": [
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)"
],
"transformOperation": "COPY: \"metadata_aspect_v2\".\"urn\" AS \"urn\"",
"confidenceScore": 0.9,
"query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29"
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)"
],
"transformOperation": "COPY: \"metadata_aspect_v2\".\"aspect\" AS \"aspect\"",
"confidenceScore": 0.9,
"query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29"
}
]
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "queryProperties",
"aspect": {
"json": {
"customProperties": {},
"statement": {
"value": "SELECT\n metadata_aspect_v2.urn,\n metadata_aspect_v2.aspect\nFROM metadata_aspect_v2\nWHERE\n (\n metadata_aspect_v2.version = 0\n )",
"language": "SQL"
},
"source": "SYSTEM",
"created": {
"time": 0,
"actor": "urn:li:corpuser:_ingestion"
},
"lastModified": {
"time": 1646575200000,
"actor": "urn:li:corpuser:_ingestion"
}
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "querySubjects",
"aspect": {
"json": {
"subjects": [
{
"entity": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)"
},
{
"entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)"
},
{
"entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)"
},
{
"entity": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)"
},
{
"entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)"
},
{
"entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)"
}
]
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:postgres"
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "glossaryTerm",
"entityUrn": "urn:li:glossaryTerm:URN",
"changeType": "UPSERT",
"aspectName": "glossaryTermKey",
"aspect": {
"json": {
"name": "URN"
}
},
"systemMetadata": {
"lastObserved": 1646575200000,
"runId": "postgres-test",
"lastRunId": "no-run-id-provided"
}
}
]