From aedf1522fb1c696e0a1634571471e6a6596a39ed Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Mon, 12 Sep 2022 23:12:52 +0530 Subject: [PATCH] feat(ingest): snowflake-beta - minor changes, tests (#5910) --- .../ingestion/source/ge_data_profiler.py | 8 +- .../ingestion/source/ge_profiling_config.py | 2 +- .../source/snowflake/snowflake_profiler.py | 12 +- .../source/snowflake/snowflake_report.py | 19 + .../source/snowflake/snowflake_schema.py | 8 +- .../source/snowflake/snowflake_usage_v2.py | 13 +- .../source/snowflake/snowflake_utils.py | 5 +- .../source/snowflake/snowflake_v2.py | 4 + .../snowflake-beta/snowflake_beta_golden.json | 940 ++++++++++++++++++ .../snowflake-beta/test_snowflake_beta.py | 280 ++++++ .../tests/unit/test_snowflake_beta_source.py | 402 ++++++++ .../tests/unit/test_usage_common.py | 8 +- 12 files changed, 1676 insertions(+), 25 deletions(-) create mode 100644 metadata-ingestion/tests/integration/snowflake-beta/snowflake_beta_golden.json create mode 100644 metadata-ingestion/tests/integration/snowflake-beta/test_snowflake_beta.py create mode 100644 metadata-ingestion/tests/unit/test_snowflake_beta_source.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py index f4667f9540..abff32c27e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py @@ -277,10 +277,10 @@ class _SingleDatasetProfiler(BasicDatasetProfilerBase): columns_to_profile = columns_to_profile[ : self.config.max_number_of_fields_to_profile ] - - self.report.report_dropped( - f"The max_number_of_fields_to_profile={self.config.max_number_of_fields_to_profile} reached. Profile of columns {self.dataset_name}({', '.join(sorted(columns_being_dropped))})" - ) + if self.config.report_dropped_profiles: + self.report.report_dropped( + f"The max_number_of_fields_to_profile={self.config.max_number_of_fields_to_profile} reached. Profile of columns {self.dataset_name}({', '.join(sorted(columns_being_dropped))})" + ) return columns_to_profile @_run_with_query_combiner diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py index 249d6c1297..1e7094df8e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py @@ -22,7 +22,7 @@ class GEProfilingConfig(ConfigModel): ) report_dropped_profiles: bool = Field( default=False, - description="If datasets which were not profiled are reported in source report or not. Set to `True` for debugging purposes.", + description="Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", ) # These settings will override the ones below. diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py index 551b5443c3..d7cebdae94 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py @@ -72,9 +72,11 @@ class SnowflakeProfiler(SnowflakeCommonMixin): platform=self.platform, profiler_args=self.get_profile_args(), ): - profile.sizeInBytes = request.table.size_in_bytes # type:ignore if profile is None: continue + profile.sizeInBytes = cast( + SnowflakeProfilerRequest, request + ).table.size_in_bytes dataset_name = request.pretty_name dataset_urn = make_dataset_urn_with_platform_instance( self.platform, @@ -153,14 +155,18 @@ class SnowflakeProfiler(SnowflakeCommonMixin): size_in_bytes is not None and size_in_bytes / (2**30) <= self.config.profiling.profile_table_size_limit - ) # Note: Profiling is not allowed is size_in_bytes is not available + ) + # Note: Profiling is not allowed is size_in_bytes is not available + # and self.config.profiling.profile_table_size_limit is set ) and ( self.config.profiling.profile_table_row_limit is None or ( rows_count is not None and rows_count <= self.config.profiling.profile_table_row_limit - ) # Note: Profiling is not allowed is rows_count is not available + ) + # Note: Profiling is not allowed is rows_count is not available + # and self.config.profiling.profile_table_row_limit is set ) ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py index 1352be4850..1b4d2e4a4f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py @@ -3,6 +3,10 @@ from datahub.ingestion.source_report.usage.snowflake_usage import SnowflakeUsage class SnowflakeV2Report(SnowflakeReport, SnowflakeUsageReport): + + schemas_scanned: int = 0 + databases_scanned: int = 0 + include_usage_stats: bool = False include_operational_stats: bool = False include_technical_schema: bool = False @@ -21,3 +25,18 @@ class SnowflakeV2Report(SnowflakeReport, SnowflakeUsageReport): num_get_columns_for_table_queries: int = 0 rows_zero_objects_modified: int = 0 + + def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: + """ + Entity could be a view or a table or a schema or a database + """ + if ent_type == "table": + self.tables_scanned += 1 + elif ent_type == "view": + self.views_scanned += 1 + elif ent_type == "schema": + self.schemas_scanned += 1 + elif ent_type == "database": + self.databases_scanned += 1 + else: + raise KeyError(f"Unknown entity {ent_type}.") diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py index 49903eb51d..5fba2a1be1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py @@ -33,7 +33,7 @@ class SnowflakeColumn: ordinal_position: int is_nullable: bool data_type: str - comment: str + comment: Optional[str] @dataclass @@ -43,7 +43,7 @@ class SnowflakeTable: last_altered: datetime size_in_bytes: int rows_count: int - comment: str + comment: Optional[str] clustering_key: str pk: Optional[SnowflakePK] = None columns: List[SnowflakeColumn] = field(default_factory=list) @@ -65,7 +65,7 @@ class SnowflakeSchema: name: str created: datetime last_altered: datetime - comment: str + comment: Optional[str] tables: List[SnowflakeTable] = field(default_factory=list) views: List[SnowflakeView] = field(default_factory=list) @@ -74,7 +74,7 @@ class SnowflakeSchema: class SnowflakeDatabase: name: str created: datetime - comment: str + comment: Optional[str] schemas: List[SnowflakeSchema] = field(default_factory=list) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py index 1054e6d7ed..97f506ebd9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_usage_v2.py @@ -190,17 +190,12 @@ class SnowflakeUsageExtractor(SnowflakeQueryMixin, SnowflakeCommonMixin): def _map_user_counts(self, user_counts: Dict) -> List[DatasetUserUsageCounts]: filtered_user_counts = [] for user_count in user_counts: - user_email = user_count.get( - "email", - "{0}@{1}".format( + user_email = user_count.get("email") + if not user_email and self.config.email_domain and user_count["user_name"]: + user_email = "{0}@{1}".format( user_count["user_name"], self.config.email_domain ).lower() - if self.config.email_domain - else None, - ) - if user_email is None or not self.config.user_email_pattern.allowed( - user_email - ): + if not user_email or not self.config.user_email_pattern.allowed(user_email): continue filtered_user_counts.append( diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py index d20c63ea3b..0ffccdee45 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py @@ -1,8 +1,9 @@ import logging -from typing import Any, Optional, Protocol +from typing import Any, Optional from snowflake.connector import SnowflakeConnection from snowflake.connector.cursor import DictCursor +from typing_extensions import Protocol from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.workunit import MetadataWorkUnit @@ -124,7 +125,7 @@ class SnowflakeCommonMixin: def get_user_identifier( self: SnowflakeCommonProtocol, user_name: str, user_email: Optional[str] ) -> str: - if user_email is not None: + if user_email: return user_email.split("@")[0] return self.snowflake_identifier(user_name) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index 69ab76d437..5dd2e71513 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -413,6 +413,8 @@ class SnowflakeV2Source( conn ) for snowflake_db in databases: + self.report.report_entity_scanned(snowflake_db.name, "database") + if not self.config.database_pattern.allowed(snowflake_db.name): self.report.report_dropped(f"{snowflake_db.name}.*") continue @@ -463,6 +465,8 @@ class SnowflakeV2Source( for snowflake_schema in snowflake_db.schemas: + self.report.report_entity_scanned(snowflake_schema.name, "schema") + if not self.config.schema_pattern.allowed(snowflake_schema.name): self.report.report_dropped(f"{db_name}.{snowflake_schema.name}.*") continue diff --git a/metadata-ingestion/tests/integration/snowflake-beta/snowflake_beta_golden.json b/metadata-ingestion/tests/integration/snowflake-beta/snowflake_beta_golden.json new file mode 100644 index 0000000000..0b62905623 --- /dev/null +++ b/metadata-ingestion/tests/integration/snowflake-beta/snowflake_beta_golden.json @@ -0,0 +1,940 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"snowflake\", \"instance\": \"PROD\", \"database\": \"test_db\"}, \"name\": \"TEST_DB\", \"description\": \"Comment for TEST_DB\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:snowflake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Database\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"snowflake\", \"instance\": \"PROD\", \"database\": \"test_db\", \"schema\": \"test_schema\"}, \"name\": \"TEST_SCHEMA\", \"description\": \"comment for TEST_DB.TEST_SCHEMA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:snowflake\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:5e359958be02ce647cd9ac196dbd4585\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"test_db.test_schema.table_1\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"name\": \"TABLE_1\", \"qualifiedName\": \"test_db.test_schema.table_1\", \"description\": \"Comment for Table\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"test_db.test_schema.table_2\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"name\": \"TABLE_2\", \"qualifiedName\": \"test_db.test_schema.table_2\", \"description\": \"Comment for Table\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"test_db.test_schema.table_3\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"name\": \"TABLE_3\", \"qualifiedName\": \"test_db.test_schema.table_3\", \"description\": \"Comment for Table\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"test_db.test_schema.table_4\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"name\": \"TABLE_4\", \"qualifiedName\": \"test_db.test_schema.table_4\", \"description\": \"Comment for Table\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"test_db.test_schema.table_5\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"name\": \"TABLE_5\", \"qualifiedName\": \"test_db.test_schema.table_5\", \"description\": \"Comment for Table\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"test_db.test_schema.table_6\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"name\": \"TABLE_6\", \"qualifiedName\": \"test_db.test_schema.table_6\", \"description\": \"Comment for Table\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"test_db.test_schema.table_7\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"name\": \"TABLE_7\", \"qualifiedName\": \"test_db.test_schema.table_7\", \"description\": \"Comment for Table\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"test_db.test_schema.table_8\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"name\": \"TABLE_8\", \"qualifiedName\": \"test_db.test_schema.table_8\", \"description\": \"Comment for Table\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"test_db.test_schema.table_9\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"name\": \"TABLE_9\", \"qualifiedName\": \"test_db.test_schema.table_9\", \"description\": \"Comment for Table\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "value": "{\"schemaName\": \"test_db.test_schema.table_10\", \"platform\": \"urn:li:dataPlatform:snowflake\", \"version\": 0, \"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"hash\": \"\", \"platformSchema\": {\"com.linkedin.schema.MySqlDDL\": {\"tableSchema\": \"\"}}, \"fields\": [{\"fieldPath\": \"col_1\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_2\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_3\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_4\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_5\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_6\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_7\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_8\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_9\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}, {\"fieldPath\": \"col_10\", \"nullable\": false, \"description\": \"Comment for column\", \"type\": {\"type\": {\"com.linkedin.schema.StringType\": {}}}, \"nativeDataType\": \"VARCHAR\", \"recursive\": false, \"isPartOfKey\": false}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"name\": \"TABLE_10\", \"qualifiedName\": \"test_db.test_schema.table_10\", \"description\": \"Comment for Table\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:94c696a054bab40b73e640a7f82e3b1c\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1654621200000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"actor\": \"urn:li:corpuser:abc\", \"operationType\": \"CREATE\", \"lastUpdatedTimestamp\": 1654144861367}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/snowflake-beta/test_snowflake_beta.py b/metadata-ingestion/tests/integration/snowflake-beta/test_snowflake_beta.py new file mode 100644 index 0000000000..39a44f0149 --- /dev/null +++ b/metadata-ingestion/tests/integration/snowflake-beta/test_snowflake_beta.py @@ -0,0 +1,280 @@ +import json +from datetime import datetime, timezone +from unittest import mock + +from freezegun import freeze_time + +from datahub.configuration.common import DynamicTypedConfig +from datahub.ingestion.run.pipeline import Pipeline +from datahub.ingestion.run.pipeline_config import PipelineConfig, SourceConfig +from datahub.ingestion.source.snowflake import snowflake_query +from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config +from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery +from tests.test_helpers import mce_helpers + +NUM_TABLES = 10 +NUM_COLS = 10 +NUM_OPS = 10 + + +def default_query_results(query): + if query == SnowflakeQuery.current_role(): + return [{"CURRENT_ROLE()": "TEST_ROLE"}] + elif query == SnowflakeQuery.current_version(): + return [{"CURRENT_VERSION()": "X.Y.Z"}] + elif query == SnowflakeQuery.current_database(): + return [{"CURRENT_DATABASE()": "TEST_DB"}] + elif query == SnowflakeQuery.current_schema(): + return [{"CURRENT_SCHEMA()": "TEST_SCHEMA"}] + elif query == SnowflakeQuery.current_warehouse(): + return [{"CURRENT_WAREHOUSE()": "TEST_WAREHOUSE"}] + elif query == SnowflakeQuery.show_databases(): + return [ + { + "name": "TEST_DB", + "created_on": datetime(2021, 6, 8, 0, 0, 0, 0), + "comment": "Comment for TEST_DB", + } + ] + elif query == SnowflakeQuery.schemas_for_database("TEST_DB"): + return [ + { + "SCHEMA_NAME": "TEST_SCHEMA", + "CREATED": datetime(2021, 6, 8, 0, 0, 0, 0), + "LAST_ALTERED": datetime(2021, 6, 8, 0, 0, 0, 0), + "COMMENT": "comment for TEST_DB.TEST_SCHEMA", + } + ] + elif query == SnowflakeQuery.tables_for_database("TEST_DB"): + return [ + { + "TABLE_SCHEMA": "TEST_SCHEMA", + "TABLE_NAME": "TABLE_{}".format(tbl_idx), + "CREATED": datetime(2021, 6, 8, 0, 0, 0, 0), + "LAST_ALTERED": datetime(2021, 6, 8, 0, 0, 0, 0), + "BYTES": 1024, + "ROW_COUNT": 10000, + "COMMENT": "Comment for Table", + "CLUSTERING_KEY": None, + } + for tbl_idx in range(1, NUM_TABLES + 1) + ] + elif query == SnowflakeQuery.tables_for_schema("TEST_SCHEMA", "TEST_DB"): + return [ + { + "TABLE_NAME": "TABLE_{}".format(tbl_idx), + "CREATED": datetime(2021, 6, 8, 0, 0, 0, 0), + "LAST_ALTERED": datetime(2021, 6, 8, 0, 0, 0, 0), + "BYTES": 1024, + "ROW_COUNT": 10000, + "COMMENT": "Comment for Table", + "CLUSTERING_KEY": None, + } + for tbl_idx in range(1, NUM_TABLES + 1) + ] + elif query == SnowflakeQuery.columns_for_schema("TEST_SCHEMA", "TEST_DB"): + return [ + { + "TABLE_CATALOG": "TEST_DB", + "TABLE_SCHEMA": "TEST_SCHEMA", + "TABLE_NAME": "TABLE_{}".format(tbl_idx), + "COLUMN_NAME": "COL_{}".format(col_idx), + "ORDINAL_POSITION": 0, + "IS_NULLABLE": "NO", + "DATA_TYPE": "VARCHAR", + "COMMENT": "Comment for column", + } + for col_idx in range(1, NUM_COLS + 1) + for tbl_idx in range(1, NUM_TABLES + 1) + ] + elif query in [ + SnowflakeQuery.columns_for_table( + "TABLE_{}".format(tbl_idx), "TEST_SCHEMA", "TEST_DB" + ) + for tbl_idx in range(1, NUM_TABLES + 1) + ]: + return [ + { + "COLUMN_NAME": "COL_{}".format(col_idx), + "ORDINAL_POSITION": 0, + "IS_NULLABLE": "NO", + "DATA_TYPE": "VARCHAR", + "COMMENT": "Comment for column", + } + for col_idx in range(1, NUM_COLS + 1) + ] + elif query in ( + SnowflakeQuery.use_database("TEST_DB"), + SnowflakeQuery.show_primary_keys_for_schema("TEST_SCHEMA", "TEST_DB"), + SnowflakeQuery.show_foreign_keys_for_schema("TEST_SCHEMA", "TEST_DB"), + ): + return [] + elif query == SnowflakeQuery.get_access_history_date_range(): + return [ + { + "MIN_TIME": datetime(2021, 6, 8, 0, 0, 0, 0), + "MAX_TIME": datetime(2022, 6, 7, 7, 17, 0, 0), + } + ] + elif query == snowflake_query.SnowflakeQuery.operational_data_for_time_window( + 1654499820000, + 1654586220000, + ): + return [ + { + "QUERY_START_TIME": datetime(2022, 6, 2, 4, 41, 1, 367000).replace( + tzinfo=timezone.utc + ), + "QUERY_TEXT": "create or replace table TABLE_{} as select * from TABLE_2 left join TABLE_3 using COL_1 left join TABLE 4 using COL2".format( + op_idx + ), + "QUERY_TYPE": "CREATE_TABLE_AS_SELECT", + "ROWS_INSERTED": 0, + "ROWS_UPDATED": 0, + "ROWS_DELETED": 0, + "BASE_OBJECTS_ACCESSED": json.dumps( + [ + { + "columns": [ + {"columnId": 0, "columnName": "COL_{}".format(col_idx)} + for col_idx in range(1, NUM_COLS + 1) + ], + "objectDomain": "Table", + "objectId": 0, + "objectName": "TEST_DB.TEST_SCHEMA.TABLE_2", + }, + { + "columns": [ + {"columnId": 0, "columnName": "COL_{}".format(col_idx)} + for col_idx in range(1, NUM_COLS + 1) + ], + "objectDomain": "Table", + "objectId": 0, + "objectName": "TEST_DB.TEST_SCHEMA.TABLE_3", + }, + { + "columns": [ + {"columnId": 0, "columnName": "COL_{}".format(col_idx)} + for col_idx in range(1, NUM_COLS + 1) + ], + "objectDomain": "Table", + "objectId": 0, + "objectName": "TEST_DB.TEST_SCHEMA.TABLE_4", + }, + ] + ), + "DIRECT_OBJECTS_ACCESSED": json.dumps( + [ + { + "columns": [ + {"columnId": 0, "columnName": "COL_{}".format(col_idx)} + for col_idx in range(1, NUM_COLS + 1) + ], + "objectDomain": "Table", + "objectId": 0, + "objectName": "TEST_DB.TEST_SCHEMA.TABLE_2", + }, + { + "columns": [ + {"columnId": 0, "columnName": "COL_{}".format(col_idx)} + for col_idx in range(1, NUM_COLS + 1) + ], + "objectDomain": "Table", + "objectId": 0, + "objectName": "TEST_DB.TEST_SCHEMA.TABLE_3", + }, + { + "columns": [ + {"columnId": 0, "columnName": "COL_{}".format(col_idx)} + for col_idx in range(1, NUM_COLS + 1) + ], + "objectDomain": "Table", + "objectId": 0, + "objectName": "TEST_DB.TEST_SCHEMA.TABLE_4", + }, + ] + ), + "OBJECTS_MODIFIED": json.dumps( + [ + { + "columns": [ + {"columnId": 0, "columnName": "COL_{}".format(col_idx)} + for col_idx in range(1, NUM_COLS + 1) + ], + "objectDomain": "Table", + "objectId": 0, + "objectName": "TEST_DB.TEST_SCHEMA.TABLE_{}".format(op_idx), + } + ] + ), + "USER_NAME": "SERVICE_ACCOUNT_TESTS_ADMIN", + "FIRST_NAME": None, + "LAST_NAME": None, + "DISPLAY_NAME": "SERVICE_ACCOUNT_TESTS_ADMIN", + "EMAIL": "abc@xyz.com", + "ROLE_NAME": "ACCOUNTADMIN", + } + for op_idx in range(1, NUM_OPS + 1) + ] + + # Unreachable code + raise Exception(f"Unknown query {query}") + + +FROZEN_TIME = "2022-06-07 17:00:00" + + +@freeze_time(FROZEN_TIME) +def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph): + test_resources_dir = pytestconfig.rootpath / "tests/integration/snowflake-beta" + + # Run the metadata ingestion pipeline. + output_file = tmp_path / "snowflake_test_events.json" + golden_file = test_resources_dir / "snowflake_beta_golden.json" + + with mock.patch("snowflake.connector.connect") as mock_connect: + sf_connection = mock.MagicMock() + sf_cursor = mock.MagicMock() + mock_connect.return_value = sf_connection + sf_connection.cursor.return_value = sf_cursor + sf_cursor.execute.side_effect = default_query_results + + pipeline = Pipeline( + config=PipelineConfig( + source=SourceConfig( + type="snowflake-beta", + config=SnowflakeV2Config( + account_id="ABC12345", + username="TST_USR", + password="TST_PWD", + include_views=False, + include_technical_schema=True, + include_table_lineage=False, + include_view_lineage=False, + include_usage_stats=False, + include_operational_stats=True, + start_time=datetime(2022, 6, 6, 7, 17, 0, 0).replace( + tzinfo=timezone.utc + ), + end_time=datetime(2022, 6, 7, 7, 17, 0, 0).replace( + tzinfo=timezone.utc + ), + ), + ), + sink=DynamicTypedConfig( + type="file", config={"filename": str(output_file)} + ), + ) + ) + pipeline.run() + pipeline.pretty_print_summary() + pipeline.raise_from_status() + + # Verify the output. + + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_file, + golden_path=golden_file, + ignore_paths=[], + ) diff --git a/metadata-ingestion/tests/unit/test_snowflake_beta_source.py b/metadata-ingestion/tests/unit/test_snowflake_beta_source.py new file mode 100644 index 0000000000..6c429fb168 --- /dev/null +++ b/metadata-ingestion/tests/unit/test_snowflake_beta_source.py @@ -0,0 +1,402 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from datahub.configuration.common import ConfigurationError, OauthConfiguration +from datahub.ingestion.api.source import SourceCapability +from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config +from datahub.ingestion.source.snowflake.snowflake_v2 import SnowflakeV2Source + + +def test_snowflake_source_throws_error_on_account_id_missing(): + with pytest.raises(ConfigurationError): + SnowflakeV2Config.parse_obj( + { + "username": "user", + "password": "password", + } + ) + + +def test_snowflake_throws_error_on_client_id_missing_if_using_oauth(): + oauth_dict = { + "provider": "microsoft", + "scopes": ["https://microsoft.com/f4b353d5-ef8d/.default"], + "client_secret": "6Hb9apkbc6HD7", + "authority_url": "https://login.microsoftonline.com/yourorganisation.com", + } + # assert that this is a valid oauth config on its own + OauthConfiguration.parse_obj(oauth_dict) + with pytest.raises(ValueError): + SnowflakeV2Config.parse_obj( + { + "account_id": "test", + "authentication_type": "OAUTH_AUTHENTICATOR", + "oauth_config": oauth_dict, + } + ) + + +def test_snowflake_throws_error_on_client_secret_missing_if_use_certificate_is_false(): + oauth_dict = { + "client_id": "882e9831-7ea51cb2b954", + "provider": "microsoft", + "scopes": ["https://microsoft.com/f4b353d5-ef8d/.default"], + "use_certificate": False, + "authority_url": "https://login.microsoftonline.com/yourorganisation.com", + } + OauthConfiguration.parse_obj(oauth_dict) + + with pytest.raises(ValueError): + SnowflakeV2Config.parse_obj( + { + "account_id": "test", + "authentication_type": "OAUTH_AUTHENTICATOR", + "oauth_config": oauth_dict, + } + ) + + +def test_snowflake_throws_error_on_encoded_oauth_private_key_missing_if_use_certificate_is_true(): + oauth_dict = { + "client_id": "882e9831-7ea51cb2b954", + "provider": "microsoft", + "scopes": ["https://microsoft.com/f4b353d5-ef8d/.default"], + "use_certificate": True, + "authority_url": "https://login.microsoftonline.com/yourorganisation.com", + "encoded_oauth_public_key": "fkdsfhkshfkjsdfiuwrwfkjhsfskfhksjf==", + } + OauthConfiguration.parse_obj(oauth_dict) + with pytest.raises(ValueError): + SnowflakeV2Config.parse_obj( + { + "account_id": "test", + "authentication_type": "OAUTH_AUTHENTICATOR", + "oauth_config": oauth_dict, + } + ) + + +def test_account_id_is_added_when_host_port_is_present(): + config = SnowflakeV2Config.parse_obj( + { + "username": "user", + "password": "password", + "host_port": "acctname", + "database_pattern": {"allow": {"^demo$"}}, + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + ) + assert config.account_id == "acctname" + + +def test_snowflake_uri_default_authentication(): + + config = SnowflakeV2Config.parse_obj( + { + "username": "user", + "password": "password", + "account_id": "acctname", + "database_pattern": {"allow": {"^demo$"}}, + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + ) + + assert ( + config.get_sql_alchemy_url() + == "snowflake://user:password@acctname/?authenticator=SNOWFLAKE&warehouse=COMPUTE_WH&role" + "=sysadmin&application=acryl_datahub" + ) + + +def test_snowflake_uri_external_browser_authentication(): + + config = SnowflakeV2Config.parse_obj( + { + "username": "user", + "account_id": "acctname", + "database_pattern": {"allow": {"^demo$"}}, + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + "authentication_type": "EXTERNAL_BROWSER_AUTHENTICATOR", + } + ) + + assert ( + config.get_sql_alchemy_url() + == "snowflake://user@acctname/?authenticator=EXTERNALBROWSER&warehouse=COMPUTE_WH&role" + "=sysadmin&application=acryl_datahub" + ) + + +def test_snowflake_uri_key_pair_authentication(): + + config = SnowflakeV2Config.parse_obj( + { + "username": "user", + "account_id": "acctname", + "database_pattern": {"allow": {"^demo$"}}, + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + "authentication_type": "KEY_PAIR_AUTHENTICATOR", + "private_key_path": "/a/random/path", + "private_key_password": "a_random_password", + } + ) + + assert ( + config.get_sql_alchemy_url() + == "snowflake://user@acctname/?authenticator=SNOWFLAKE_JWT&warehouse=COMPUTE_WH&role" + "=sysadmin&application=acryl_datahub" + ) + + +def test_options_contain_connect_args(): + config = SnowflakeV2Config.parse_obj( + { + "username": "user", + "password": "password", + "host_port": "acctname", + "database_pattern": {"allow": {"^demo$"}}, + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + ) + connect_args = config.get_options().get("connect_args") + assert connect_args is not None + + +@patch("snowflake.connector.connect") +def test_test_connection_failure(mock_connect): + mock_connect.side_effect = Exception("Failed to connect to snowflake") + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + report = SnowflakeV2Source.test_connection(config) + assert report is not None + assert report.basic_connectivity + assert not report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason + assert "Failed to connect to snowflake" in report.basic_connectivity.failure_reason + + +@patch("snowflake.connector.connect") +def test_test_connection_basic_success(mock_connect): + + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + report = SnowflakeV2Source.test_connection(config) + assert report is not None + assert report.basic_connectivity + assert report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason is None + + +def setup_mock_connect(mock_connect, query_results=None): + def default_query_results(query): + if query == "select current_role()": + return [("TEST_ROLE",)] + elif query == "select current_secondary_roles()": + return [('{"roles":"","value":""}',)] + elif query == "select current_warehouse()": + return [("TEST_WAREHOUSE")] + # Unreachable code + raise Exception() + + connection_mock = MagicMock() + cursor_mock = MagicMock() + cursor_mock.execute.side_effect = ( + query_results if query_results is not None else default_query_results + ) + connection_mock.cursor.return_value = cursor_mock + mock_connect.return_value = connection_mock + + +@patch("snowflake.connector.connect") +def test_test_connection_no_warehouse(mock_connect): + def query_results(query): + if query == "select current_role()": + return [("TEST_ROLE",)] + elif query == "select current_secondary_roles()": + return [('{"roles":"","value":""}',)] + elif query == "select current_warehouse()": + return [(None,)] + elif query == 'show grants to role "TEST_ROLE"': + return [ + ("", "USAGE", "DATABASE", "DB1"), + ("", "USAGE", "SCHEMA", "DB1.SCHEMA1"), + ("", "REFERENCES", "TABLE", "DB1.SCHEMA1.TABLE1"), + ] + elif query == 'show grants to role "PUBLIC"': + return [] + # Unreachable code + raise Exception() + + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + setup_mock_connect(mock_connect, query_results) + report = SnowflakeV2Source.test_connection(config) + assert report is not None + assert report.basic_connectivity + assert report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason is None + + assert report.capability_report + assert report.capability_report[SourceCapability.CONTAINERS].capable + assert not report.capability_report[SourceCapability.SCHEMA_METADATA].capable + failure_reason = report.capability_report[ + SourceCapability.SCHEMA_METADATA + ].failure_reason + assert failure_reason + + assert "Current role does not have permissions to use warehouse" in failure_reason + + +@patch("snowflake.connector.connect") +def test_test_connection_capability_schema_failure(mock_connect): + def query_results(query): + if query == "select current_role()": + return [("TEST_ROLE",)] + elif query == "select current_secondary_roles()": + return [('{"roles":"","value":""}',)] + elif query == "select current_warehouse()": + return [("TEST_WAREHOUSE",)] + elif query == 'show grants to role "TEST_ROLE"': + return [("", "USAGE", "DATABASE", "DB1")] + elif query == 'show grants to role "PUBLIC"': + return [] + # Unreachable code + raise Exception() + + setup_mock_connect(mock_connect, query_results) + + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + report = SnowflakeV2Source.test_connection(config) + assert report is not None + assert report.basic_connectivity + assert report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason is None + assert report.capability_report + + assert report.capability_report[SourceCapability.CONTAINERS].capable + assert not report.capability_report[SourceCapability.SCHEMA_METADATA].capable + assert ( + report.capability_report[SourceCapability.SCHEMA_METADATA].failure_reason + is not None + ) + + +@patch("snowflake.connector.connect") +def test_test_connection_capability_schema_success(mock_connect): + def query_results(query): + if query == "select current_role()": + return [("TEST_ROLE",)] + elif query == "select current_secondary_roles()": + return [('{"roles":"","value":""}',)] + elif query == "select current_warehouse()": + return [("TEST_WAREHOUSE")] + elif query == 'show grants to role "TEST_ROLE"': + return [ + ["", "USAGE", "DATABASE", "DB1"], + ["", "USAGE", "SCHEMA", "DB1.SCHEMA1"], + ["", "REFERENCES", "TABLE", "DB1.SCHEMA1.TABLE1"], + ] + elif query == 'show grants to role "PUBLIC"': + return [] + # Unreachable code + raise Exception() + + setup_mock_connect(mock_connect, query_results) + + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + report = SnowflakeV2Source.test_connection(config) + + assert report is not None + assert report.basic_connectivity + assert report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason is None + assert report.capability_report + + assert report.capability_report[SourceCapability.CONTAINERS].capable + assert report.capability_report[SourceCapability.SCHEMA_METADATA].capable + assert report.capability_report[SourceCapability.DESCRIPTIONS].capable + + +@patch("snowflake.connector.connect") +def test_test_connection_capability_all_success(mock_connect): + def query_results(query): + if query == "select current_role()": + return [("TEST_ROLE",)] + elif query == "select current_secondary_roles()": + return [('{"roles":"","value":""}',)] + elif query == "select current_warehouse()": + return [("TEST_WAREHOUSE")] + elif query == 'show grants to role "TEST_ROLE"': + return [ + ("", "USAGE", "DATABASE", "DB1"), + ("", "USAGE", "SCHEMA", "DB1.SCHEMA1"), + ("", "SELECT", "TABLE", "DB1.SCHEMA1.TABLE1"), + ("", "USAGE", "ROLE", "TEST_USAGE_ROLE"), + ] + elif query == 'show grants to role "PUBLIC"': + return [] + elif query == 'show grants to role "TEST_USAGE_ROLE"': + return [ + ["", "USAGE", "DATABASE", "SNOWFLAKE"], + ["", "USAGE", "SCHEMA", "ACCOUNT_USAGE"], + ["", "USAGE", "VIEW", "SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY"], + ["", "USAGE", "VIEW", "SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY"], + ["", "USAGE", "VIEW", "SNOWFLAKE.ACCOUNT_USAGE.OBJECT_DEPENDENCIES"], + ] + # Unreachable code + raise Exception() + + setup_mock_connect(mock_connect, query_results) + + config = { + "username": "user", + "password": "password", + "account_id": "missing", + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + } + report = SnowflakeV2Source.test_connection(config) + assert report is not None + assert report.basic_connectivity + assert report.basic_connectivity.capable + assert report.basic_connectivity.failure_reason is None + assert report.capability_report + + assert report.capability_report[SourceCapability.CONTAINERS].capable + assert report.capability_report[SourceCapability.SCHEMA_METADATA].capable + assert report.capability_report[SourceCapability.DATA_PROFILING].capable + assert report.capability_report[SourceCapability.DESCRIPTIONS].capable + assert report.capability_report[SourceCapability.LINEAGE_COARSE].capable diff --git a/metadata-ingestion/tests/unit/test_usage_common.py b/metadata-ingestion/tests/unit/test_usage_common.py index 91b9e1a551..1fdc712623 100644 --- a/metadata-ingestion/tests/unit/test_usage_common.py +++ b/metadata-ingestion/tests/unit/test_usage_common.py @@ -1,4 +1,5 @@ from datetime import datetime +from unittest import mock import pytest from pydantic import ValidationError @@ -238,8 +239,11 @@ def test_query_trimming(): def test_top_n_queries_validator_fails(): with pytest.raises(ValidationError) as excinfo: - GenericAggregatedDataset.total_budget_for_query_list = 20 - BaseUsageConfig(top_n_queries=2) + with mock.patch( + "datahub.ingestion.source.usage.usage_common.GenericAggregatedDataset.total_budget_for_query_list", + 20, + ): + BaseUsageConfig(top_n_queries=2) assert "top_n_queries is set to 2 but it can be maximum 1" in str(excinfo.value)