diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py index e1798f5d86..3b8cd8d328 100644 --- a/metadata-ingestion/src/datahub/emitter/mce_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py @@ -5,7 +5,7 @@ import re import time from enum import Enum from hashlib import md5 -from typing import Any, List, Optional, Type, TypeVar, Union, cast, get_type_hints +from typing import Any, List, Optional, Set, Type, TypeVar, Union, cast, get_type_hints import typing_inspect from avrogen.dict_wrapper import DictWrapper @@ -35,7 +35,20 @@ from datahub.metadata.schema_classes import ( DEFAULT_ENV = DEFAULT_ENV_CONFIGURATION DEFAULT_FLOW_CLUSTER = "prod" UNKNOWN_USER = "urn:li:corpuser:unknown" - +SQL_STYLE_PLATFORMS: Set[str] = { + "athena", + "bigquery", + "druid", + "hive", + "mariadb", + "mssql", + "mysql", + "oracle", + "postgres", + "redshift", + "snowflake", + "trino", +} logger = logging.getLogger(__name__) @@ -57,6 +70,9 @@ def make_data_platform_urn(platform: str) -> str: def make_dataset_urn(platform: str, name: str, env: str = DEFAULT_ENV) -> str: + # Use lower-case name for all SQL style datasets + if platform in SQL_STYLE_PLATFORMS: + name = name.lower() return f"urn:li:dataset:({make_data_platform_urn(platform)},{name},{env})" @@ -70,7 +86,12 @@ def make_dataplatform_instance_urn(platform: str, instance: str) -> str: def make_dataset_urn_with_platform_instance( platform: str, name: str, platform_instance: Optional[str], env: str = DEFAULT_ENV ) -> str: + if platform_instance: + # Use lower-case name for all SQL style datasets + if platform in SQL_STYLE_PLATFORMS: + name = name.lower() + return f"urn:li:dataset:({make_data_platform_urn(platform)},{platform_instance}.{name},{env})" else: return make_dataset_urn(platform=platform, name=name, env=env) @@ -190,7 +211,6 @@ def make_domain_urn(domain: str) -> str: def make_ml_primary_key_urn(feature_table_name: str, primary_key_name: str) -> str: - return f"urn:li:mlPrimaryKey:({feature_table_name},{primary_key_name})" @@ -198,7 +218,6 @@ def make_ml_feature_urn( feature_table_name: str, feature_name: str, ) -> str: - return f"urn:li:mlFeature:({feature_table_name},{feature_name})" diff --git a/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json b/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json index 7e4a118788..560ceb191a 100644 --- a/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json +++ b/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json @@ -1,5 +1,5 @@ [ - { +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", @@ -7,18 +7,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\"}, \"name\": \"demodata\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\"}, \"name\": \"demodata\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", @@ -26,18 +26,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", @@ -45,18 +45,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Database\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Database\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", @@ -64,18 +64,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_accessadmin\"}, \"name\": \"db_accessadmin\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_accessadmin\"}, \"name\": \"db_accessadmin\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", @@ -83,18 +83,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", @@ -102,18 +102,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", @@ -121,18 +121,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", @@ -140,18 +140,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_backupoperator\"}, \"name\": \"db_backupoperator\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_backupoperator\"}, \"name\": \"db_backupoperator\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", @@ -159,18 +159,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", @@ -178,18 +178,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", @@ -197,18 +197,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", @@ -216,18 +216,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_datareader\"}, \"name\": \"db_datareader\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_datareader\"}, \"name\": \"db_datareader\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", @@ -235,18 +235,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", @@ -254,18 +254,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", @@ -273,18 +273,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", @@ -292,18 +292,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_datawriter\"}, \"name\": \"db_datawriter\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_datawriter\"}, \"name\": \"db_datawriter\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", @@ -311,18 +311,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", @@ -330,18 +330,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", @@ -349,18 +349,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", @@ -368,18 +368,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_ddladmin\"}, \"name\": \"db_ddladmin\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_ddladmin\"}, \"name\": \"db_ddladmin\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", @@ -387,18 +387,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", @@ -406,18 +406,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", @@ -425,18 +425,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", @@ -444,18 +444,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_denydatareader\"}, \"name\": \"db_denydatareader\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_denydatareader\"}, \"name\": \"db_denydatareader\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", @@ -463,18 +463,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", @@ -482,18 +482,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", @@ -501,18 +501,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", @@ -520,18 +520,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_denydatawriter\"}, \"name\": \"db_denydatawriter\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_denydatawriter\"}, \"name\": \"db_denydatawriter\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", @@ -539,18 +539,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", @@ -558,18 +558,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", @@ -577,18 +577,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", @@ -596,18 +596,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_owner\"}, \"name\": \"db_owner\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_owner\"}, \"name\": \"db_owner\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", @@ -615,18 +615,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", @@ -634,18 +634,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", @@ -653,18 +653,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", @@ -672,18 +672,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_securityadmin\"}, \"name\": \"db_securityadmin\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_securityadmin\"}, \"name\": \"db_securityadmin\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", @@ -691,18 +691,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", @@ -710,18 +710,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", @@ -729,18 +729,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", @@ -748,18 +748,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"dbo\"}, \"name\": \"dbo\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"dbo\"}, \"name\": \"dbo\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", @@ -767,18 +767,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", @@ -786,18 +786,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", @@ -805,125 +805,125 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.dbo.products,PROD)", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d41a036a2e6cfa44b834edf7683199ec\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:d41a036a2e6cfa44b834edf7683199ec\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "DemoData.dbo.Products", - "platform": "urn:li:dataPlatform:mssql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.dbo.products,PROD)", + "aspects": [ { - "fieldPath": "ID", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.common.Status": { + "removed": false } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null }, { - "fieldPath": "ProductName", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.dbo.Products", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "ProductName", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "NVARCHAR()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - }, - "nativeDataType": "NVARCHAR()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", @@ -931,18 +931,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"Foo\"}, \"name\": \"Foo\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"Foo\"}, \"name\": \"Foo\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", @@ -950,18 +950,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", @@ -969,18 +969,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", @@ -988,384 +988,384 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "DemoData.Foo.Items", - "platform": "urn:li:dataPlatform:mssql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", + "aspects": [ { - "fieldPath": "ID", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.common.Status": { + "removed": false } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null }, { - "fieldPath": "ItemName", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.Items", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "ItemName", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "NVARCHAR()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null } - }, - "nativeDataType": "NVARCHAR()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "DemoData.Foo.Persons", - "platform": "urn:li:dataPlatform:mssql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.Persons", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null + }, + { + "fieldPath": "LastName", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "FirstName", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Age", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } } - }, - "fields": [ - { - "fieldPath": "ID", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "LastName", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "FirstName", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Age", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "DemoData.Foo.SalesReason", - "platform": "urn:li:dataPlatform:mssql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)", + "aspects": [ { - "fieldPath": "TempID", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.common.Status": { + "removed": false } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null }, { - "fieldPath": "Name", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.SalesReason", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "TempID", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null + }, + { + "fieldPath": "Name", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "NVARCHAR(length=50)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "FK_TempSales_SalesReason", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),TempID)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)" + } + ] } - }, - "nativeDataType": "NVARCHAR(length=50)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "FK_TempSales_SalesReason", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD),ID)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD),TempID)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)" - } - ] - } - } - ] - } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", @@ -1373,18 +1373,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"guest\"}, \"name\": \"guest\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"guest\"}, \"name\": \"guest\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", @@ -1392,18 +1392,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", @@ -1411,18 +1411,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", @@ -1430,18 +1430,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", @@ -1449,18 +1449,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"INFORMATION_SCHEMA\"}, \"name\": \"INFORMATION_SCHEMA\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"INFORMATION_SCHEMA\"}, \"name\": \"INFORMATION_SCHEMA\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", @@ -1468,18 +1468,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", @@ -1487,18 +1487,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", @@ -1506,18 +1506,18 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", @@ -1525,18 +1525,18 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"sys\"}, \"name\": \"sys\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"sys\"}, \"name\": \"sys\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", @@ -1544,18 +1544,18 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", @@ -1563,18 +1563,18 @@ "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - }, - { +}, +{ "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", @@ -1582,15 +1582,15 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } - } +} ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json index 8fce02d856..f3154f8fcc 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json @@ -67791,7 +67791,7 @@ "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { - "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.activitiesAll,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.contacts,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.activitiesall,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.contacts,PROD)\", \"type\": \"TRANSFORMED\"}]}", "contentType": "application/json" }, "systemMetadata": { @@ -87188,7 +87188,7 @@ "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.activitiesAll,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.activitiesall,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.BrowsePaths": { @@ -87703,7 +87703,7 @@ "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.contacts,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.contacts,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.BrowsePaths": { @@ -104195,7 +104195,7 @@ "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.activitiesAll,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.activitiesall,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.BrowsePaths": { @@ -104710,7 +104710,7 @@ "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.contacts,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.contacts,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.BrowsePaths": { @@ -124265,7 +124265,7 @@ "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.activitiesAll,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.activitiesall,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.BrowsePaths": { @@ -124780,7 +124780,7 @@ "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.contacts,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.contacts,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.BrowsePaths": { diff --git a/metadata-ingestion/tests/unit/test_mce_builder.py b/metadata-ingestion/tests/unit/test_mce_builder.py index d4c00cbc53..e313fa3208 100644 --- a/metadata-ingestion/tests/unit/test_mce_builder.py +++ b/metadata-ingestion/tests/unit/test_mce_builder.py @@ -1,3 +1,7 @@ +from typing import Dict, Optional, Tuple + +import pytest + import datahub.emitter.mce_builder as builder from datahub.metadata.schema_classes import ( DataFlowInfoClass, @@ -21,3 +25,83 @@ def test_can_add_aspect(): assert builder.can_add_aspect(dataset_mce, DatasetPropertiesClass) assert builder.can_add_aspect(dataset_mce, OwnershipClass) assert not builder.can_add_aspect(dataset_mce, DataFlowInfoClass) + + +test_make_dataset_urns_params: Dict[ + str, Tuple[Tuple[str, str, Optional[str], str], str] +] = { + "athena": ( + ("athena", "ATABLE", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:athena,MY_INSTANCE.atable,PROD)", + ), + "bigquery": ( + ("bigquery", "ATable", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:bigquery,MY_INSTANCE.atable,PROD)", + ), + "bigquery_no_instance": ( + ("bigquery", "ATable", None, "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:bigquery,atable,PROD)", + ), + "druid": ( + ("druid", "AtaBLE", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:druid,MY_INSTANCE.atable,PROD)", + ), + "hive": ( + ("hive", "ataBLE", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:hive,MY_INSTANCE.atable,PROD)", + ), + "mariadb": ( + ("mariadb", "aTAble", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:mariadb,MY_INSTANCE.atable,PROD)", + ), + "mssql": ( + ("mssql", "aTAblE", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:mssql,MY_INSTANCE.atable,PROD)", + ), + "mysql": ( + ("mysql", "aTABlE", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:mysql,MY_INSTANCE.atable,PROD)", + ), + "oracle": ( + ("oracle", "AtAbLe", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:oracle,MY_INSTANCE.atable,PROD)", + ), + "postgres": ( + ("postgres", "AtAbLE", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:postgres,MY_INSTANCE.atable,PROD)", + ), + "redshift": ( + ("redshift", "atAbLE", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:redshift,MY_INSTANCE.atable,PROD)", + ), + "snowflake": ( + ("snowflake", "atABle", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:snowflake,MY_INSTANCE.atable,PROD)", + ), + "trino": ( + ("trino", "AtaBle", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:trino,MY_INSTANCE.atable,PROD)", + ), + "kafka_no_lower_casing": ( + ("kafka", "MyKafkaTopic", "MY_INSTANCE", "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:kafka,MY_INSTANCE.MyKafkaTopic,PROD)", + ), + "kafka_no_instance_no_lower_casing": ( + ("kafka", "MyKafkaTopic", None, "PROD"), + "urn:li:dataset:(urn:li:dataPlatform:kafka,MyKafkaTopic,PROD)", + ), +} + + +@pytest.mark.parametrize( + "urnParts, expected", + test_make_dataset_urns_params.values(), + ids=test_make_dataset_urns_params.keys(), +) +def test_make_dataset_urns( + urnParts: Tuple[str, str, Optional[str], str], expected: str +) -> None: + dataset_urn = builder.make_dataset_urn_with_platform_instance( + urnParts[0], urnParts[1], urnParts[2], urnParts[3] + ) + assert dataset_urn == expected diff --git a/metadata-ingestion/tests/unit/test_redash_source.py b/metadata-ingestion/tests/unit/test_redash_source.py index 95d368a949..525b67a655 100644 --- a/metadata-ingestion/tests/unit/test_redash_source.py +++ b/metadata-ingestion/tests/unit/test_redash_source.py @@ -524,7 +524,7 @@ def test_get_known_viz_chart_snapshot(mocked_data_source): ), ), chartUrl="http://localhost:5000/queries/4#10", - inputs=["urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam,PROD)"], + inputs=["urn:li:dataset:(urn:li:dataPlatform:mysql,rfam,PROD)"], type="PIE", ) ], @@ -557,7 +557,7 @@ def test_get_unknown_viz_chart_snapshot(mocked_data_source): ), ), chartUrl="http://localhost:5000/queries/4#9", - inputs=["urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam,PROD)"], + inputs=["urn:li:dataset:(urn:li:dataPlatform:mysql,rfam,PROD)"], type="TABLE", ) ], @@ -685,9 +685,9 @@ def test_get_chart_snapshot_parse_table_names_from_sql(mocked_data_source): ), chartUrl="http://localhost:5000/queries/4#10", inputs=[ - "urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.order_items,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.orders,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.staffs,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.order_items,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.orders,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.staffs,PROD)", ], type="PIE", ) diff --git a/smoke-test/test_data_quality.py b/smoke-test/test_data_quality.py index d753748eef..544fd98987 100644 --- a/smoke-test/test_data_quality.py +++ b/smoke-test/test_data_quality.py @@ -1,17 +1,31 @@ import json -import urllib import time +import urllib + import pytest import requests from datahub.cli.docker import check_local_docker_containers -from tests.utils import ingest_file_via_rest -from datahub.metadata.schema_classes import AssertionResultTypeClass, DatasetAssertionInfoClass, PartitionTypeClass, AssertionInfoClass, AssertionTypeClass, DatasetAssertionScopeClass, DatasetColumnAssertionClass, AssertionStdOperatorClass, DatasetColumnStdAggFuncClass, AssertionRunEventClass, PartitionSpecClass, AssertionResultClass, AssertionRunStatusClass -from datahub.emitter.mce_builder import make_schema_field_urn, make_dataset_urn +from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.common import PipelineContext, RecordEnvelope +from datahub.ingestion.api.sink import NoopWriteCallback from datahub.ingestion.sink.file import FileSink, FileSinkConfig -from datahub.ingestion.api.sink import NoopWriteCallback, WriteCallback -from datahub.ingestion.api.common import RecordEnvelope -from datahub.ingestion.api.common import PipelineContext +from datahub.metadata.schema_classes import (AssertionInfoClass, + AssertionResultClass, + AssertionResultTypeClass, + AssertionRunEventClass, + AssertionRunStatusClass, + AssertionStdOperatorClass, + AssertionTypeClass, + DatasetAssertionInfoClass, + DatasetAssertionScopeClass, + DatasetColumnAssertionClass, + DatasetColumnStdAggFuncClass, + PartitionSpecClass, + PartitionTypeClass) + +from tests.utils import ingest_file_via_rest + GMS_ENDPOINT = "http://localhost:8080" restli_default_headers = { @@ -21,37 +35,39 @@ restli_default_headers = { def create_test_data(test_file): assertion_urn = "urn:li:assertion:2d3b06a6e77e1f24adc9860a05ea089b" - dataset_urn=make_dataset_urn(platform="postgres", name="fooTable") + dataset_urn = make_dataset_urn(platform="postgres", name="fooTable") assertion_info = AssertionInfoClass( type=AssertionTypeClass.DATASET, - customProperties={ - "suite_name": "demo_suite" - }, + customProperties={"suite_name": "demo_suite"}, datasetAssertion=DatasetAssertionInfoClass( - fields=[make_schema_field_urn(dataset_urn,"col1")], + fields=[make_schema_field_urn(dataset_urn, "col1")], datasets=[dataset_urn], scope=DatasetAssertionScopeClass.DATASET_COLUMN, columnAssertion=DatasetColumnAssertionClass( - stdOperator=AssertionStdOperatorClass.LESS_THAN, - nativeOperator="column_value_is_less_than", - stdAggFunc=DatasetColumnStdAggFuncClass.IDENTITY, + stdOperator=AssertionStdOperatorClass.LESS_THAN, + nativeOperator="column_value_is_less_than", + stdAggFunc=DatasetColumnStdAggFuncClass.IDENTITY, + ), ), - ), - parameters={ - "max_value": "99" - }, - ) + parameters={"max_value": "99"}, + ) # The assertion definition mcp1 = MetadataChangeProposalWrapper( entityType="assertion", changeType="UPSERT", entityUrn=assertion_urn, aspectName="assertionInfo", - aspect=assertion_info + aspect=assertion_info, ) - timestamps = [1643794280350, 1643794280352, 1643794280354, 1643880726872, 1643880726874, 1643880726875] - msg_ids = [] - # The assertion run event attached to the dataset + timestamps = [ + 1643794280350, + 1643794280352, + 1643794280354, + 1643880726872, + 1643880726874, + 1643880726875, + ] + # The assertion run event attached to the dataset mcp2 = MetadataChangeProposalWrapper( entityType="dataset", entityUrn=dataset_urn, @@ -62,7 +78,7 @@ def create_test_data(test_file): partitionSpec=PartitionSpecClass( partition="[{'country': 'IN'}]", type=PartitionTypeClass.PARTITION, - ), + ), messageId=str(timestamps[0]), assertionUrn=assertion_urn, asserteeUrn=dataset_urn, @@ -70,10 +86,10 @@ def create_test_data(test_file): type=AssertionResultTypeClass.SUCCESS, actualAggValue=90, externalUrl="http://example.com/uuid1", - ), + ), runId="uuid1", - status=AssertionRunStatusClass.COMPLETE - ) + status=AssertionRunStatusClass.COMPLETE, + ), ) mcp3 = MetadataChangeProposalWrapper( @@ -86,7 +102,7 @@ def create_test_data(test_file): partitionSpec=PartitionSpecClass( partition="[{'country': 'US'}]", type=PartitionTypeClass.PARTITION, - ), + ), messageId=str(timestamps[1]), assertionUrn=assertion_urn, asserteeUrn=dataset_urn, @@ -94,10 +110,10 @@ def create_test_data(test_file): type=AssertionResultTypeClass.FAILURE, actualAggValue=101, externalUrl="http://example.com/uuid1", - ), + ), runId="uuid1", - status=AssertionRunStatusClass.COMPLETE - ) + status=AssertionRunStatusClass.COMPLETE, + ), ) # Result of evaluating this assertion on the whole dataset mcp4 = MetadataChangeProposalWrapper( @@ -110,7 +126,7 @@ def create_test_data(test_file): partitionSpec=PartitionSpecClass( partition="FULL_TABLE_SNAPSHOT", type=PartitionTypeClass.FULL_TABLE, - ), + ), messageId=str(timestamps[2]), assertionUrn=assertion_urn, asserteeUrn=dataset_urn, @@ -118,10 +134,10 @@ def create_test_data(test_file): type=AssertionResultTypeClass.SUCCESS, actualAggValue=93, externalUrl="http://example.com/uuid1", - ), + ), runId="uuid1", - status=AssertionRunStatusClass.COMPLETE - ) + status=AssertionRunStatusClass.COMPLETE, + ), ) mcp5 = MetadataChangeProposalWrapper( @@ -134,7 +150,7 @@ def create_test_data(test_file): partitionSpec=PartitionSpecClass( partition="[{'country': 'IN'}]", type=PartitionTypeClass.PARTITION, - ), + ), messageId=str(timestamps[3]), assertionUrn=assertion_urn, asserteeUrn=dataset_urn, @@ -142,10 +158,10 @@ def create_test_data(test_file): type=AssertionResultTypeClass.SUCCESS, actualAggValue=90, externalUrl="http://example.com/uuid1", - ), + ), runId="uuid1", - status=AssertionRunStatusClass.COMPLETE - ) + status=AssertionRunStatusClass.COMPLETE, + ), ) mcp6 = MetadataChangeProposalWrapper( entityType="dataset", @@ -157,7 +173,7 @@ def create_test_data(test_file): partitionSpec=PartitionSpecClass( partition="[{'country': 'US'}]", type=PartitionTypeClass.PARTITION, - ), + ), messageId=str(timestamps[4]), assertionUrn=assertion_urn, asserteeUrn=dataset_urn, @@ -165,10 +181,10 @@ def create_test_data(test_file): type=AssertionResultTypeClass.FAILURE, actualAggValue=101, externalUrl="http://example.com/uuid1", - ), + ), runId="uuid1", - status=AssertionRunStatusClass.COMPLETE - ) + status=AssertionRunStatusClass.COMPLETE, + ), ) # Result of evaluating this assertion on the whole dataset @@ -182,7 +198,7 @@ def create_test_data(test_file): partitionSpec=PartitionSpecClass( partition="FULL_TABLE_SNAPSHOT", type=PartitionTypeClass.FULL_TABLE, - ), + ), messageId=str(timestamps[5]), assertionUrn=assertion_urn, asserteeUrn=dataset_urn, @@ -190,20 +206,22 @@ def create_test_data(test_file): type=AssertionResultTypeClass.SUCCESS, actualAggValue=93, externalUrl="http://example.com/uuid1", - ), + ), runId="uuid1", - status=AssertionRunStatusClass.COMPLETE - ) + status=AssertionRunStatusClass.COMPLETE, + ), ) - fileSink: FileSink = FileSink.create(FileSinkConfig(filename=test_file), ctx=PipelineContext(run_id="test-file")) + fileSink: FileSink = FileSink.create( + FileSinkConfig(filename=test_file), ctx=PipelineContext(run_id="test-file") + ) for mcp in [mcp1, mcp2, mcp3, mcp4, mcp5, mcp6, mcp7]: - fileSink.write_record_async(RecordEnvelope(record=mcp, metadata={}), write_callback=NoopWriteCallback()) + fileSink.write_record_async( + RecordEnvelope(record=mcp, metadata={}), write_callback=NoopWriteCallback() + ) fileSink.close() - - @pytest.fixture(scope="session") def generate_test_data(tmp_path_factory): """Generates metadata events data and stores into a test file""" @@ -211,7 +229,7 @@ def generate_test_data(tmp_path_factory): file_name = dir_name / "test_dq_events.json" create_test_data(test_file=str(file_name)) yield str(file_name) - + @pytest.fixture(scope="session") def wait_for_healthchecks(generate_test_data): @@ -233,7 +251,7 @@ def test_run_ingestion(wait_for_healthchecks, generate_test_data): @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_gms_get_latest_assertions_results_by_partition(): - urn = "urn:li:dataset:(urn:li:dataPlatform:postgres,fooTable,PROD)" + urn = "urn:li:dataset:(urn:li:dataPlatform:postgres,footable,PROD)" # sleep for elasticsearch indices to be updated time.sleep(5) @@ -258,9 +276,7 @@ def test_gms_get_latest_assertions_results_by_partition(): } ] }, - "metrics": [ - {"fieldPath": "result", "aggregationType": "LATEST"} - ], + "metrics": [{"fieldPath": "result", "aggregationType": "LATEST"}], "buckets": [ {"key": "asserteeUrn", "type": "STRING_GROUPING_BUCKET"}, {"key": "partitionSpec.partition", "type": "STRING_GROUPING_BUCKET"}, @@ -303,7 +319,7 @@ def test_gms_get_latest_assertions_results_by_partition(): @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_gms_get_assertions_on_dataset(): """lists all assertion urns including those which may not have executed""" - urn = "urn:li:dataset:(urn:li:dataPlatform:postgres,fooTable,PROD)" + urn = "urn:li:dataset:(urn:li:dataPlatform:postgres,footable,PROD)" response = requests.get( f"{GMS_ENDPOINT}/relationships?direction=INCOMING&urn={urllib.parse.quote(urn)}&types=Asserts" ) @@ -316,7 +332,7 @@ def test_gms_get_assertions_on_dataset(): @pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"]) def test_gms_get_assertions_on_dataset_field(): """lists all assertion urns including those which may not have executed""" - dataset_urn=make_dataset_urn("postgres","fooTable") + dataset_urn = make_dataset_urn("postgres", "fooTable") field_urn = make_schema_field_urn(dataset_urn, "col1") response = requests.get( f"{GMS_ENDPOINT}/relationships?direction=INCOMING&urn={urllib.parse.quote(field_urn)}&types=Asserts" @@ -342,5 +358,6 @@ def test_gms_get_assertion_info(): assert data["aspect"] assert data["aspect"]["com.linkedin.assertion.AssertionInfo"] assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["type"] == "DATASET" - assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["datasetAssertion"]["scope"] - + assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["datasetAssertion"][ + "scope" + ]