yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-01 11:19:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(snowflake): avoid reporting warnings/info for sys tables (#11114)

Browse Source
This commit is contained in:
Harshal Sheth 2024-08-07 14:04:18 -07:00 committed by GitHub
parent c226883097
commit a25df8e6a0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/src/datahub/ingestion/api/source.py
View File

@ -47,7 +47,7 @@ from datahub.utilities.type_annotations import get_class_from_annotation
logger = logging.getLogger(__name__)
_MAX_CONTEXT_STRING_LENGTH = 300
_MAX_CONTEXT_STRING_LENGTH = 1000
class SourceCapability(Enum):

2
metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py
View File

@ -440,7 +440,7 @@ class SnowflakeSchemaGenerator(SnowflakeStructuredReportMixin):
yield from self._process_tag(tag)
if not snowflake_schema.views and not snowflake_schema.tables:
self.structured_reporter.warning(
self.structured_reporter.info(
title="No tables/views found in schema",
message="If tables exist, please grant REFERENCES or SELECT permissions on them.",
context=f"{db_name}.{schema_name}",

20
metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py
View File

@ -127,6 +127,8 @@ class SnowflakeFilter:
SnowflakeObjectDomain.MATERIALIZED_VIEW,
):
return False
if _is_sys_table(dataset_name):
return False
if len(dataset_params) != 3:
self.structured_reporter.info(
@ -176,6 +178,11 @@ def _combine_identifier_parts(
return f"{db_name}.{schema_name}.{table_name}"
def _is_sys_table(table_name: str) -> bool:
# Often will look like `SYS$_UNPIVOT_VIEW1737` or `sys$_pivot_view19`.
return table_name.lower().startswith("sys$")
# Qualified Object names from snowflake audit logs have quotes for for snowflake quoted identifiers,
# For example "test-database"."test-schema".test_table
# whereas we generate urns without quotes even for quoted identifiers for backward compatibility
@ -186,12 +193,13 @@ def _cleanup_qualified_name(
) -> str:
name_parts = qualified_name.split(".")
if len(name_parts) != 3:
structured_reporter.info(
title="Unexpected dataset pattern",
message="We failed to parse a Snowflake qualified name into its constituent parts. "
"DB/schema/table filtering may not work as expected on these entities.",
context=f"{qualified_name} has {len(name_parts)} parts",
)
if not _is_sys_table(qualified_name):
structured_reporter.info(
title="Unexpected dataset pattern",
message="We failed to parse a Snowflake qualified name into its constituent parts. "
"DB/schema/table filtering may not work as expected on these entities.",
context=f"{qualified_name} has {len(name_parts)} parts",
)
return qualified_name.replace('"', "")
return _combine_identifier_parts(
db_name=name_parts[0].strip('"'),