mirror of
https://github.com/datahub-project/datahub.git
synced 2025-08-19 14:44:37 +00:00
fix(ingest): Use lower-case dataset names in the dataset urns for all SQL-styled datasets. (#4140)
This commit is contained in:
Ravindra Lanka
GitHub
parent
ede6d91534
commit
6c75185445
@ -5,7 +5,7 @@ import re
|
||||
import time
|
||||
from enum import Enum
|
||||
from hashlib import md5
|
||||
from typing import Any, List, Optional, Type, TypeVar, Union, cast, get_type_hints
|
||||
from typing import Any, List, Optional, Set, Type, TypeVar, Union, cast, get_type_hints
|
||||
|
||||
import typing_inspect
|
||||
from avrogen.dict_wrapper import DictWrapper
|
||||
@ -35,7 +35,20 @@ from datahub.metadata.schema_classes import (
|
||||
DEFAULT_ENV = DEFAULT_ENV_CONFIGURATION
|
||||
DEFAULT_FLOW_CLUSTER = "prod"
|
||||
UNKNOWN_USER = "urn:li:corpuser:unknown"
|
||||
|
||||
SQL_STYLE_PLATFORMS: Set[str] = {
|
||||
"athena",
|
||||
"bigquery",
|
||||
"druid",
|
||||
"hive",
|
||||
"mariadb",
|
||||
"mssql",
|
||||
"mysql",
|
||||
"oracle",
|
||||
"postgres",
|
||||
"redshift",
|
||||
"snowflake",
|
||||
"trino",
|
||||
}
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -57,6 +70,9 @@ def make_data_platform_urn(platform: str) -> str:
|
||||
|
||||
|
||||
def make_dataset_urn(platform: str, name: str, env: str = DEFAULT_ENV) -> str:
|
||||
# Use lower-case name for all SQL style datasets
|
||||
if platform in SQL_STYLE_PLATFORMS:
|
||||
name = name.lower()
|
||||
return f"urn:li:dataset:({make_data_platform_urn(platform)},{name},{env})"
|
||||
|
||||
|
||||
@ -70,7 +86,12 @@ def make_dataplatform_instance_urn(platform: str, instance: str) -> str:
|
||||
def make_dataset_urn_with_platform_instance(
|
||||
platform: str, name: str, platform_instance: Optional[str], env: str = DEFAULT_ENV
|
||||
) -> str:
|
||||
|
||||
if platform_instance:
|
||||
# Use lower-case name for all SQL style datasets
|
||||
if platform in SQL_STYLE_PLATFORMS:
|
||||
name = name.lower()
|
||||
|
||||
return f"urn:li:dataset:({make_data_platform_urn(platform)},{platform_instance}.{name},{env})"
|
||||
else:
|
||||
return make_dataset_urn(platform=platform, name=name, env=env)
|
||||
@ -190,7 +211,6 @@ def make_domain_urn(domain: str) -> str:
|
||||
|
||||
|
||||
def make_ml_primary_key_urn(feature_table_name: str, primary_key_name: str) -> str:
|
||||
|
||||
return f"urn:li:mlPrimaryKey:({feature_table_name},{primary_key_name})"
|
||||
|
||||
|
||||
@ -198,7 +218,6 @@ def make_ml_feature_urn(
|
||||
feature_table_name: str,
|
||||
feature_name: str,
|
||||
) -> str:
|
||||
|
||||
return f"urn:li:mlFeature:({feature_table_name},{feature_name})"
|
||||
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
[
|
||||
{
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
|
||||
@ -17,8 +17,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
|
||||
@ -36,8 +36,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5",
|
||||
@ -55,8 +55,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec",
|
||||
@ -74,8 +74,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec",
|
||||
@ -93,8 +93,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec",
|
||||
@ -112,8 +112,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec",
|
||||
@ -131,8 +131,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0",
|
||||
@ -150,8 +150,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0",
|
||||
@ -169,8 +169,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0",
|
||||
@ -188,8 +188,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0",
|
||||
@ -207,8 +207,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee",
|
||||
@ -226,8 +226,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee",
|
||||
@ -245,8 +245,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee",
|
||||
@ -264,8 +264,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee",
|
||||
@ -283,8 +283,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e",
|
||||
@ -302,8 +302,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e",
|
||||
@ -321,8 +321,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e",
|
||||
@ -340,8 +340,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e",
|
||||
@ -359,8 +359,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590",
|
||||
@ -378,8 +378,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590",
|
||||
@ -397,8 +397,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590",
|
||||
@ -416,8 +416,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590",
|
||||
@ -435,8 +435,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782",
|
||||
@ -454,8 +454,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782",
|
||||
@ -473,8 +473,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782",
|
||||
@ -492,8 +492,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782",
|
||||
@ -511,8 +511,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30",
|
||||
@ -530,8 +530,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30",
|
||||
@ -549,8 +549,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30",
|
||||
@ -568,8 +568,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30",
|
||||
@ -587,8 +587,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92",
|
||||
@ -606,8 +606,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92",
|
||||
@ -625,8 +625,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92",
|
||||
@ -644,8 +644,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92",
|
||||
@ -663,8 +663,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809",
|
||||
@ -682,8 +682,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809",
|
||||
@ -701,8 +701,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809",
|
||||
@ -720,8 +720,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809",
|
||||
@ -739,8 +739,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec",
|
||||
@ -758,8 +758,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec",
|
||||
@ -777,8 +777,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec",
|
||||
@ -796,8 +796,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec",
|
||||
@ -815,11 +815,11 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.dbo.products,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "container",
|
||||
@ -834,12 +834,12 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.dbo.products,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.Status": {
|
||||
@ -922,8 +922,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
|
||||
@ -941,8 +941,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
|
||||
@ -960,8 +960,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
|
||||
@ -979,8 +979,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671",
|
||||
@ -998,11 +998,11 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "container",
|
||||
@ -1017,12 +1017,12 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.Status": {
|
||||
@ -1105,11 +1105,11 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "container",
|
||||
@ -1124,12 +1124,12 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.Status": {
|
||||
@ -1246,11 +1246,11 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "container",
|
||||
@ -1265,12 +1265,12 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.Status": {
|
||||
@ -1343,12 +1343,12 @@
|
||||
{
|
||||
"name": "FK_TempSales_SalesReason",
|
||||
"foreignFields": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD),ID)"
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)"
|
||||
],
|
||||
"sourceFields": [
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD),TempID)"
|
||||
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),TempID)"
|
||||
],
|
||||
"foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)"
|
||||
"foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1364,8 +1364,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac",
|
||||
@ -1383,8 +1383,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac",
|
||||
@ -1402,8 +1402,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac",
|
||||
@ -1421,8 +1421,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac",
|
||||
@ -1440,8 +1440,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8",
|
||||
@ -1459,8 +1459,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8",
|
||||
@ -1478,8 +1478,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8",
|
||||
@ -1497,8 +1497,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8",
|
||||
@ -1516,8 +1516,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
|
||||
@ -1535,8 +1535,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
|
||||
@ -1554,8 +1554,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
|
||||
@ -1573,8 +1573,8 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "container",
|
||||
"entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f",
|
||||
@ -1592,5 +1592,5 @@
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -67791,7 +67791,7 @@
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "upstreamLineage",
|
||||
"aspect": {
|
||||
"value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.activitiesAll,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.contacts,PROD)\", \"type\": \"TRANSFORMED\"}]}",
|
||||
"value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.activitiesall,PROD)\", \"type\": \"TRANSFORMED\"}, {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.contacts,PROD)\", \"type\": \"TRANSFORMED\"}]}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
@ -87188,7 +87188,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.activitiesAll,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.activitiesall,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.BrowsePaths": {
|
||||
@ -87703,7 +87703,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.contacts,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.contacts,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.BrowsePaths": {
|
||||
@ -104195,7 +104195,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.activitiesAll,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.activitiesall,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.BrowsePaths": {
|
||||
@ -104710,7 +104710,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.contacts,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.contacts,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.BrowsePaths": {
|
||||
@ -124265,7 +124265,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.activitiesAll,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.activitiesall,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.BrowsePaths": {
|
||||
@ -124780,7 +124780,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,Oracle Eloqua.contacts,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oracle eloqua.contacts,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.common.BrowsePaths": {
|
||||
|
||||
@ -1,3 +1,7 @@
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
import pytest
|
||||
|
||||
import datahub.emitter.mce_builder as builder
|
||||
from datahub.metadata.schema_classes import (
|
||||
DataFlowInfoClass,
|
||||
@ -21,3 +25,83 @@ def test_can_add_aspect():
|
||||
assert builder.can_add_aspect(dataset_mce, DatasetPropertiesClass)
|
||||
assert builder.can_add_aspect(dataset_mce, OwnershipClass)
|
||||
assert not builder.can_add_aspect(dataset_mce, DataFlowInfoClass)
|
||||
|
||||
|
||||
test_make_dataset_urns_params: Dict[
|
||||
str, Tuple[Tuple[str, str, Optional[str], str], str]
|
||||
] = {
|
||||
"athena": (
|
||||
("athena", "ATABLE", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:athena,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"bigquery": (
|
||||
("bigquery", "ATable", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:bigquery,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"bigquery_no_instance": (
|
||||
("bigquery", "ATable", None, "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:bigquery,atable,PROD)",
|
||||
),
|
||||
"druid": (
|
||||
("druid", "AtaBLE", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:druid,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"hive": (
|
||||
("hive", "ataBLE", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:hive,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"mariadb": (
|
||||
("mariadb", "aTAble", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:mariadb,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"mssql": (
|
||||
("mssql", "aTAblE", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:mssql,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"mysql": (
|
||||
("mysql", "aTABlE", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:mysql,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"oracle": (
|
||||
("oracle", "AtAbLe", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:oracle,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"postgres": (
|
||||
("postgres", "AtAbLE", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:postgres,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"redshift": (
|
||||
("redshift", "atAbLE", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:redshift,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"snowflake": (
|
||||
("snowflake", "atABle", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:snowflake,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"trino": (
|
||||
("trino", "AtaBle", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:trino,MY_INSTANCE.atable,PROD)",
|
||||
),
|
||||
"kafka_no_lower_casing": (
|
||||
("kafka", "MyKafkaTopic", "MY_INSTANCE", "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:kafka,MY_INSTANCE.MyKafkaTopic,PROD)",
|
||||
),
|
||||
"kafka_no_instance_no_lower_casing": (
|
||||
("kafka", "MyKafkaTopic", None, "PROD"),
|
||||
"urn:li:dataset:(urn:li:dataPlatform:kafka,MyKafkaTopic,PROD)",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"urnParts, expected",
|
||||
test_make_dataset_urns_params.values(),
|
||||
ids=test_make_dataset_urns_params.keys(),
|
||||
)
|
||||
def test_make_dataset_urns(
|
||||
urnParts: Tuple[str, str, Optional[str], str], expected: str
|
||||
) -> None:
|
||||
dataset_urn = builder.make_dataset_urn_with_platform_instance(
|
||||
urnParts[0], urnParts[1], urnParts[2], urnParts[3]
|
||||
)
|
||||
assert dataset_urn == expected
|
||||
|
||||
@ -524,7 +524,7 @@ def test_get_known_viz_chart_snapshot(mocked_data_source):
|
||||
),
|
||||
),
|
||||
chartUrl="http://localhost:5000/queries/4#10",
|
||||
inputs=["urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam,PROD)"],
|
||||
inputs=["urn:li:dataset:(urn:li:dataPlatform:mysql,rfam,PROD)"],
|
||||
type="PIE",
|
||||
)
|
||||
],
|
||||
@ -557,7 +557,7 @@ def test_get_unknown_viz_chart_snapshot(mocked_data_source):
|
||||
),
|
||||
),
|
||||
chartUrl="http://localhost:5000/queries/4#9",
|
||||
inputs=["urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam,PROD)"],
|
||||
inputs=["urn:li:dataset:(urn:li:dataPlatform:mysql,rfam,PROD)"],
|
||||
type="TABLE",
|
||||
)
|
||||
],
|
||||
@ -685,9 +685,9 @@ def test_get_chart_snapshot_parse_table_names_from_sql(mocked_data_source):
|
||||
),
|
||||
chartUrl="http://localhost:5000/queries/4#10",
|
||||
inputs=[
|
||||
"urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.order_items,PROD)",
|
||||
"urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.orders,PROD)",
|
||||
"urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.staffs,PROD)",
|
||||
"urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.order_items,PROD)",
|
||||
"urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.orders,PROD)",
|
||||
"urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.staffs,PROD)",
|
||||
],
|
||||
type="PIE",
|
||||
)
|
||||
|
||||
@ -1,17 +1,31 @@
|
||||
import json
|
||||
import urllib
|
||||
import time
|
||||
import urllib
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from datahub.cli.docker import check_local_docker_containers
|
||||
from tests.utils import ingest_file_via_rest
|
||||
from datahub.metadata.schema_classes import AssertionResultTypeClass, DatasetAssertionInfoClass, PartitionTypeClass, AssertionInfoClass, AssertionTypeClass, DatasetAssertionScopeClass, DatasetColumnAssertionClass, AssertionStdOperatorClass, DatasetColumnStdAggFuncClass, AssertionRunEventClass, PartitionSpecClass, AssertionResultClass, AssertionRunStatusClass
|
||||
from datahub.emitter.mce_builder import make_schema_field_urn, make_dataset_urn
|
||||
from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn
|
||||
from datahub.emitter.mcp import MetadataChangeProposalWrapper
|
||||
from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
|
||||
from datahub.ingestion.api.sink import NoopWriteCallback
|
||||
from datahub.ingestion.sink.file import FileSink, FileSinkConfig
|
||||
from datahub.ingestion.api.sink import NoopWriteCallback, WriteCallback
|
||||
from datahub.ingestion.api.common import RecordEnvelope
|
||||
from datahub.ingestion.api.common import PipelineContext
|
||||
from datahub.metadata.schema_classes import (AssertionInfoClass,
|
||||
AssertionResultClass,
|
||||
AssertionResultTypeClass,
|
||||
AssertionRunEventClass,
|
||||
AssertionRunStatusClass,
|
||||
AssertionStdOperatorClass,
|
||||
AssertionTypeClass,
|
||||
DatasetAssertionInfoClass,
|
||||
DatasetAssertionScopeClass,
|
||||
DatasetColumnAssertionClass,
|
||||
DatasetColumnStdAggFuncClass,
|
||||
PartitionSpecClass,
|
||||
PartitionTypeClass)
|
||||
|
||||
from tests.utils import ingest_file_via_rest
|
||||
|
||||
GMS_ENDPOINT = "http://localhost:8080"
|
||||
|
||||
restli_default_headers = {
|
||||
@ -21,14 +35,12 @@ restli_default_headers = {
|
||||
|
||||
def create_test_data(test_file):
|
||||
assertion_urn = "urn:li:assertion:2d3b06a6e77e1f24adc9860a05ea089b"
|
||||
dataset_urn=make_dataset_urn(platform="postgres", name="fooTable")
|
||||
dataset_urn = make_dataset_urn(platform="postgres", name="fooTable")
|
||||
assertion_info = AssertionInfoClass(
|
||||
type=AssertionTypeClass.DATASET,
|
||||
customProperties={
|
||||
"suite_name": "demo_suite"
|
||||
},
|
||||
customProperties={"suite_name": "demo_suite"},
|
||||
datasetAssertion=DatasetAssertionInfoClass(
|
||||
fields=[make_schema_field_urn(dataset_urn,"col1")],
|
||||
fields=[make_schema_field_urn(dataset_urn, "col1")],
|
||||
datasets=[dataset_urn],
|
||||
scope=DatasetAssertionScopeClass.DATASET_COLUMN,
|
||||
columnAssertion=DatasetColumnAssertionClass(
|
||||
@ -37,9 +49,7 @@ def create_test_data(test_file):
|
||||
stdAggFunc=DatasetColumnStdAggFuncClass.IDENTITY,
|
||||
),
|
||||
),
|
||||
parameters={
|
||||
"max_value": "99"
|
||||
},
|
||||
parameters={"max_value": "99"},
|
||||
)
|
||||
# The assertion definition
|
||||
mcp1 = MetadataChangeProposalWrapper(
|
||||
@ -47,10 +57,16 @@ def create_test_data(test_file):
|
||||
changeType="UPSERT",
|
||||
entityUrn=assertion_urn,
|
||||
aspectName="assertionInfo",
|
||||
aspect=assertion_info
|
||||
aspect=assertion_info,
|
||||
)
|
||||
timestamps = [1643794280350, 1643794280352, 1643794280354, 1643880726872, 1643880726874, 1643880726875]
|
||||
msg_ids = []
|
||||
timestamps = [
|
||||
1643794280350,
|
||||
1643794280352,
|
||||
1643794280354,
|
||||
1643880726872,
|
||||
1643880726874,
|
||||
1643880726875,
|
||||
]
|
||||
# The assertion run event attached to the dataset
|
||||
mcp2 = MetadataChangeProposalWrapper(
|
||||
entityType="dataset",
|
||||
@ -72,8 +88,8 @@ def create_test_data(test_file):
|
||||
externalUrl="http://example.com/uuid1",
|
||||
),
|
||||
runId="uuid1",
|
||||
status=AssertionRunStatusClass.COMPLETE
|
||||
)
|
||||
status=AssertionRunStatusClass.COMPLETE,
|
||||
),
|
||||
)
|
||||
|
||||
mcp3 = MetadataChangeProposalWrapper(
|
||||
@ -96,8 +112,8 @@ def create_test_data(test_file):
|
||||
externalUrl="http://example.com/uuid1",
|
||||
),
|
||||
runId="uuid1",
|
||||
status=AssertionRunStatusClass.COMPLETE
|
||||
)
|
||||
status=AssertionRunStatusClass.COMPLETE,
|
||||
),
|
||||
)
|
||||
# Result of evaluating this assertion on the whole dataset
|
||||
mcp4 = MetadataChangeProposalWrapper(
|
||||
@ -120,8 +136,8 @@ def create_test_data(test_file):
|
||||
externalUrl="http://example.com/uuid1",
|
||||
),
|
||||
runId="uuid1",
|
||||
status=AssertionRunStatusClass.COMPLETE
|
||||
)
|
||||
status=AssertionRunStatusClass.COMPLETE,
|
||||
),
|
||||
)
|
||||
|
||||
mcp5 = MetadataChangeProposalWrapper(
|
||||
@ -144,8 +160,8 @@ def create_test_data(test_file):
|
||||
externalUrl="http://example.com/uuid1",
|
||||
),
|
||||
runId="uuid1",
|
||||
status=AssertionRunStatusClass.COMPLETE
|
||||
)
|
||||
status=AssertionRunStatusClass.COMPLETE,
|
||||
),
|
||||
)
|
||||
mcp6 = MetadataChangeProposalWrapper(
|
||||
entityType="dataset",
|
||||
@ -167,8 +183,8 @@ def create_test_data(test_file):
|
||||
externalUrl="http://example.com/uuid1",
|
||||
),
|
||||
runId="uuid1",
|
||||
status=AssertionRunStatusClass.COMPLETE
|
||||
)
|
||||
status=AssertionRunStatusClass.COMPLETE,
|
||||
),
|
||||
)
|
||||
|
||||
# Result of evaluating this assertion on the whole dataset
|
||||
@ -192,18 +208,20 @@ def create_test_data(test_file):
|
||||
externalUrl="http://example.com/uuid1",
|
||||
),
|
||||
runId="uuid1",
|
||||
status=AssertionRunStatusClass.COMPLETE
|
||||
)
|
||||
status=AssertionRunStatusClass.COMPLETE,
|
||||
),
|
||||
)
|
||||
|
||||
fileSink: FileSink = FileSink.create(FileSinkConfig(filename=test_file), ctx=PipelineContext(run_id="test-file"))
|
||||
fileSink: FileSink = FileSink.create(
|
||||
FileSinkConfig(filename=test_file), ctx=PipelineContext(run_id="test-file")
|
||||
)
|
||||
for mcp in [mcp1, mcp2, mcp3, mcp4, mcp5, mcp6, mcp7]:
|
||||
fileSink.write_record_async(RecordEnvelope(record=mcp, metadata={}), write_callback=NoopWriteCallback())
|
||||
fileSink.write_record_async(
|
||||
RecordEnvelope(record=mcp, metadata={}), write_callback=NoopWriteCallback()
|
||||
)
|
||||
fileSink.close()
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def generate_test_data(tmp_path_factory):
|
||||
"""Generates metadata events data and stores into a test file"""
|
||||
@ -233,7 +251,7 @@ def test_run_ingestion(wait_for_healthchecks, generate_test_data):
|
||||
|
||||
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
|
||||
def test_gms_get_latest_assertions_results_by_partition():
|
||||
urn = "urn:li:dataset:(urn:li:dataPlatform:postgres,fooTable,PROD)"
|
||||
urn = "urn:li:dataset:(urn:li:dataPlatform:postgres,footable,PROD)"
|
||||
|
||||
# sleep for elasticsearch indices to be updated
|
||||
time.sleep(5)
|
||||
@ -258,9 +276,7 @@ def test_gms_get_latest_assertions_results_by_partition():
|
||||
}
|
||||
]
|
||||
},
|
||||
"metrics": [
|
||||
{"fieldPath": "result", "aggregationType": "LATEST"}
|
||||
],
|
||||
"metrics": [{"fieldPath": "result", "aggregationType": "LATEST"}],
|
||||
"buckets": [
|
||||
{"key": "asserteeUrn", "type": "STRING_GROUPING_BUCKET"},
|
||||
{"key": "partitionSpec.partition", "type": "STRING_GROUPING_BUCKET"},
|
||||
@ -303,7 +319,7 @@ def test_gms_get_latest_assertions_results_by_partition():
|
||||
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
|
||||
def test_gms_get_assertions_on_dataset():
|
||||
"""lists all assertion urns including those which may not have executed"""
|
||||
urn = "urn:li:dataset:(urn:li:dataPlatform:postgres,fooTable,PROD)"
|
||||
urn = "urn:li:dataset:(urn:li:dataPlatform:postgres,footable,PROD)"
|
||||
response = requests.get(
|
||||
f"{GMS_ENDPOINT}/relationships?direction=INCOMING&urn={urllib.parse.quote(urn)}&types=Asserts"
|
||||
)
|
||||
@ -316,7 +332,7 @@ def test_gms_get_assertions_on_dataset():
|
||||
@pytest.mark.dependency(depends=["test_healthchecks", "test_run_ingestion"])
|
||||
def test_gms_get_assertions_on_dataset_field():
|
||||
"""lists all assertion urns including those which may not have executed"""
|
||||
dataset_urn=make_dataset_urn("postgres","fooTable")
|
||||
dataset_urn = make_dataset_urn("postgres", "fooTable")
|
||||
field_urn = make_schema_field_urn(dataset_urn, "col1")
|
||||
response = requests.get(
|
||||
f"{GMS_ENDPOINT}/relationships?direction=INCOMING&urn={urllib.parse.quote(field_urn)}&types=Asserts"
|
||||
@ -342,5 +358,6 @@ def test_gms_get_assertion_info():
|
||||
assert data["aspect"]
|
||||
assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]
|
||||
assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["type"] == "DATASET"
|
||||
assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["datasetAssertion"]["scope"]
|
||||
|
||||
assert data["aspect"]["com.linkedin.assertion.AssertionInfo"]["datasetAssertion"][
|
||||
"scope"
|
||||
]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user