datahub/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json

[
{
    "entityType": "container",
    "entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff",
    "changeType": "UPSERT",
    "aspectName": "containerProperties",
    "aspect": {
        "json": {
            "customProperties": {
                "platform": "postgres",
                "env": "PROD",
                "database": "postgrestest"
            },
            "name": "postgrestest",
            "env": "PROD"
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "container",
    "entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff",
    "changeType": "UPSERT",
    "aspectName": "status",
    "aspect": {
        "json": {
            "removed": false
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "container",
    "entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff",
    "changeType": "UPSERT",
    "aspectName": "dataPlatformInstance",
    "aspect": {
        "json": {
            "platform": "urn:li:dataPlatform:postgres"
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "container",
    "entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff",
    "changeType": "UPSERT",
    "aspectName": "subTypes",
    "aspect": {
        "json": {
            "typeNames": [
                "Database"
            ]
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "container",
    "entityUrn": "urn:li:container:a6097853edba03be190d99ece4b307ff",
    "changeType": "UPSERT",
    "aspectName": "browsePathsV2",
    "aspect": {
        "json": {
            "path": []
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "container",
    "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
    "changeType": "UPSERT",
    "aspectName": "container",
    "aspect": {
        "json": {
            "container": "urn:li:container:a6097853edba03be190d99ece4b307ff"
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "container",
    "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
    "changeType": "UPSERT",
    "aspectName": "containerProperties",
    "aspect": {
        "json": {
            "customProperties": {
                "platform": "postgres",
                "env": "PROD",
                "database": "postgrestest",
                "schema": "public"
            },
            "name": "public",
            "env": "PROD"
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "container",
    "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
    "changeType": "UPSERT",
    "aspectName": "status",
    "aspect": {
        "json": {
            "removed": false
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "container",
    "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
    "changeType": "UPSERT",
    "aspectName": "dataPlatformInstance",
    "aspect": {
        "json": {
            "platform": "urn:li:dataPlatform:postgres"
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "container",
    "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
    "changeType": "UPSERT",
    "aspectName": "subTypes",
    "aspect": {
        "json": {
            "typeNames": [
                "Schema"
            ]
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "container",
    "entityUrn": "urn:li:container:51904fc8cd5cc729bc630decff284525",
    "changeType": "UPSERT",
    "aspectName": "browsePathsV2",
    "aspect": {
        "json": {
            "path": [
                {
                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
                }
            ]
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
    "changeType": "UPSERT",
    "aspectName": "container",
    "aspect": {
        "json": {
            "container": "urn:li:container:51904fc8cd5cc729bc630decff284525"
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "proposedSnapshot": {
        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
            "aspects": [
                {
                    "com.linkedin.pegasus2avro.common.Status": {
                        "removed": false
                    }
                },
                {
                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                        "customProperties": {},
                        "name": "metadata_aspect_v2",
                        "tags": []
                    }
                },
                {
                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
                        "schemaName": "postgrestest.public.metadata_aspect_v2",
                        "platform": "urn:li:dataPlatform:postgres",
                        "version": 0,
                        "created": {
                            "time": 0,
                            "actor": "urn:li:corpuser:unknown"
                        },
                        "lastModified": {
                            "time": 0,
                            "actor": "urn:li:corpuser:unknown"
                        },
                        "hash": "",
                        "platformSchema": {
                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
                                "tableSchema": ""
                            }
                        },
                        "fields": [
                            {
                                "fieldPath": "urn",
                                "nullable": false,
                                "type": {
                                    "type": {
                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                    }
                                },
                                "nativeDataType": "VARCHAR(500)",
                                "recursive": false,
                                "glossaryTerms": {
                                    "terms": [
                                        {
                                            "urn": "urn:li:glossaryTerm:URN"
                                        }
                                    ],
                                    "auditStamp": {
                                        "time": 1646575200000,
                                        "actor": "urn:li:corpuser:datahub"
                                    }
                                },
                                "isPartOfKey": true
                            },
                            {
                                "fieldPath": "aspect",
                                "nullable": false,
                                "type": {
                                    "type": {
                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                    }
                                },
                                "nativeDataType": "VARCHAR(200)",
                                "recursive": false,
                                "isPartOfKey": true
                            },
                            {
                                "fieldPath": "version",
                                "nullable": false,
                                "type": {
                                    "type": {
                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
                                    }
                                },
                                "nativeDataType": "BIGINT",
                                "recursive": false,
                                "isPartOfKey": true
                            },
                            {
                                "fieldPath": "metadata",
                                "nullable": false,
                                "type": {
                                    "type": {
                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                    }
                                },
                                "nativeDataType": "TEXT",
                                "recursive": false,
                                "isPartOfKey": false
                            },
                            {
                                "fieldPath": "systemmetadata",
                                "nullable": true,
                                "type": {
                                    "type": {
                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                    }
                                },
                                "nativeDataType": "TEXT",
                                "recursive": false,
                                "isPartOfKey": false
                            },
                            {
                                "fieldPath": "createdon",
                                "nullable": false,
                                "type": {
                                    "type": {
                                        "com.linkedin.pegasus2avro.schema.TimeType": {}
                                    }
                                },
                                "nativeDataType": "TIMESTAMP WITHOUT TIME ZONE",
                                "recursive": false,
                                "isPartOfKey": false
                            },
                            {
                                "fieldPath": "createdby",
                                "nullable": false,
                                "type": {
                                    "type": {
                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                    }
                                },
                                "nativeDataType": "VARCHAR(255)",
                                "recursive": false,
                                "glossaryTerms": {
                                    "terms": [
                                        {
                                            "urn": "urn:li:glossaryTerm:URN"
                                        }
                                    ],
                                    "auditStamp": {
                                        "time": 1646575200000,
                                        "actor": "urn:li:corpuser:datahub"
                                    }
                                },
                                "isPartOfKey": false
                            },
                            {
                                "fieldPath": "createdfor",
                                "nullable": true,
                                "type": {
                                    "type": {
                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                    }
                                },
                                "nativeDataType": "VARCHAR(255)",
                                "recursive": false,
                                "isPartOfKey": false
                            },
                            {
                                "fieldPath": "metadata_json",
                                "nullable": false,
                                "type": {
                                    "type": {
                                        "com.linkedin.pegasus2avro.schema.RecordType": {}
                                    }
                                },
                                "nativeDataType": "JSON",
                                "recursive": false,
                                "isPartOfKey": false
                            }
                        ]
                    }
                }
            ]
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
    "changeType": "UPSERT",
    "aspectName": "subTypes",
    "aspect": {
        "json": {
            "typeNames": [
                "Table"
            ]
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
    "changeType": "UPSERT",
    "aspectName": "browsePathsV2",
    "aspect": {
        "json": {
            "path": [
                {
                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
                },
                {
                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
                }
            ]
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
    "changeType": "UPSERT",
    "aspectName": "container",
    "aspect": {
        "json": {
            "container": "urn:li:container:51904fc8cd5cc729bc630decff284525"
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "proposedSnapshot": {
        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
            "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
            "aspects": [
                {
                    "com.linkedin.pegasus2avro.common.Status": {
                        "removed": false
                    }
                },
                {
                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
                        "customProperties": {
                            "is_view": "True",
                            "view_definition": " SELECT metadata_aspect_v2.urn,\n    metadata_aspect_v2.aspect\n   FROM metadata_aspect_v2\n  WHERE (metadata_aspect_v2.version = 0);"
                        },
                        "name": "metadata_aspect_view",
                        "tags": []
                    }
                },
                {
                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
                        "schemaName": "postgrestest.public.metadata_aspect_view",
                        "platform": "urn:li:dataPlatform:postgres",
                        "version": 0,
                        "created": {
                            "time": 0,
                            "actor": "urn:li:corpuser:unknown"
                        },
                        "lastModified": {
                            "time": 0,
                            "actor": "urn:li:corpuser:unknown"
                        },
                        "hash": "",
                        "platformSchema": {
                            "com.linkedin.pegasus2avro.schema.MySqlDDL": {
                                "tableSchema": ""
                            }
                        },
                        "fields": [
                            {
                                "fieldPath": "urn",
                                "nullable": true,
                                "type": {
                                    "type": {
                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                    }
                                },
                                "nativeDataType": "VARCHAR(500)",
                                "recursive": false,
                                "isPartOfKey": false
                            },
                            {
                                "fieldPath": "aspect",
                                "nullable": true,
                                "type": {
                                    "type": {
                                        "com.linkedin.pegasus2avro.schema.StringType": {}
                                    }
                                },
                                "nativeDataType": "VARCHAR(200)",
                                "recursive": false,
                                "isPartOfKey": false
                            }
                        ]
                    }
                }
            ]
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
    "changeType": "UPSERT",
    "aspectName": "subTypes",
    "aspect": {
        "json": {
            "typeNames": [
                "View"
            ]
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
    "changeType": "UPSERT",
    "aspectName": "viewProperties",
    "aspect": {
        "json": {
            "materialized": false,
            "viewLogic": " SELECT metadata_aspect_v2.urn,\n    metadata_aspect_v2.aspect\n   FROM metadata_aspect_v2\n  WHERE (metadata_aspect_v2.version = 0);",
            "viewLanguage": "SQL"
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
    "changeType": "UPSERT",
    "aspectName": "browsePathsV2",
    "aspect": {
        "json": {
            "path": [
                {
                    "id": "urn:li:container:a6097853edba03be190d99ece4b307ff",
                    "urn": "urn:li:container:a6097853edba03be190d99ece4b307ff"
                },
                {
                    "id": "urn:li:container:51904fc8cd5cc729bc630decff284525",
                    "urn": "urn:li:container:51904fc8cd5cc729bc630decff284525"
                }
            ]
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
    "changeType": "UPSERT",
    "aspectName": "datasetProfile",
    "aspect": {
        "json": {
            "timestampMillis": 1646575200000,
            "partitionSpec": {
                "partition": "FULL_TABLE_SNAPSHOT",
                "type": "FULL_TABLE"
            },
            "rowCount": 2,
            "columnCount": 9,
            "fieldProfiles": [],
            "sizeInBytes": 16384
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "dataset",
    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)",
    "changeType": "UPSERT",
    "aspectName": "upstreamLineage",
    "aspect": {
        "json": {
            "upstreams": [
                {
                    "auditStamp": {
                        "time": 1646575200000,
                        "actor": "urn:li:corpuser:_ingestion"
                    },
                    "created": {
                        "time": 0,
                        "actor": "urn:li:corpuser:_ingestion"
                    },
                    "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)",
                    "type": "VIEW",
                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29"
                }
            ],
            "fineGrainedLineages": [
                {
                    "upstreamType": "FIELD_SET",
                    "upstreams": [
                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)"
                    ],
                    "downstreamType": "FIELD",
                    "downstreams": [
                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)"
                    ],
                    "transformOperation": "COPY: \"metadata_aspect_v2\".\"urn\" AS \"urn\"",
                    "confidenceScore": 0.9,
                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29"
                },
                {
                    "upstreamType": "FIELD_SET",
                    "upstreams": [
                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)"
                    ],
                    "downstreamType": "FIELD",
                    "downstreams": [
                        "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)"
                    ],
                    "transformOperation": "COPY: \"metadata_aspect_v2\".\"aspect\" AS \"aspect\"",
                    "confidenceScore": 0.9,
                    "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29"
                }
            ]
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "query",
    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29",
    "changeType": "UPSERT",
    "aspectName": "queryProperties",
    "aspect": {
        "json": {
            "customProperties": {},
            "statement": {
                "value": "SELECT\n  metadata_aspect_v2.urn,\n  metadata_aspect_v2.aspect\nFROM metadata_aspect_v2\nWHERE\n  (\n    metadata_aspect_v2.version = 0\n  )",
                "language": "SQL"
            },
            "source": "SYSTEM",
            "created": {
                "time": 0,
                "actor": "urn:li:corpuser:_ingestion"
            },
            "lastModified": {
                "time": 1646575200000,
                "actor": "urn:li:corpuser:_ingestion"
            }
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "query",
    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29",
    "changeType": "UPSERT",
    "aspectName": "querySubjects",
    "aspect": {
        "json": {
            "subjects": [
                {
                    "entity": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD)"
                },
                {
                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),aspect)"
                },
                {
                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_v2,PROD),urn)"
                },
                {
                    "entity": "urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD)"
                },
                {
                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),urn)"
                },
                {
                    "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,postgrestest.public.metadata_aspect_view,PROD),aspect)"
                }
            ]
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "query",
    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29",
    "changeType": "UPSERT",
    "aspectName": "dataPlatformInstance",
    "aspect": {
        "json": {
            "platform": "urn:li:dataPlatform:postgres"
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "query",
    "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Apostgres%2Cpostgrestest.public.metadata_aspect_view%2CPROD%29",
    "changeType": "UPSERT",
    "aspectName": "status",
    "aspect": {
        "json": {
            "removed": false
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
},
{
    "entityType": "glossaryTerm",
    "entityUrn": "urn:li:glossaryTerm:URN",
    "changeType": "UPSERT",
    "aspectName": "glossaryTermKey",
    "aspect": {
        "json": {
            "name": "URN"
        }
    },
    "systemMetadata": {
        "lastObserved": 1646575200000,
        "runId": "postgres-test",
        "lastRunId": "no-run-id-provided"
    }
}
]