diff --git a/ingestion/src/metadata/ingestion/lineage/masker.py b/ingestion/src/metadata/ingestion/lineage/masker.py index 69aab2d7ba0..e5578393405 100644 --- a/ingestion/src/metadata/ingestion/lineage/masker.py +++ b/ingestion/src/metadata/ingestion/lineage/masker.py @@ -127,4 +127,4 @@ def mask_query( except Exception as exc: logger.debug(f"Failed to mask query with sqlfluff: {exc}") logger.debug(traceback.format_exc()) - return query + return None diff --git a/ingestion/src/metadata/ingestion/lineage/parser.py b/ingestion/src/metadata/ingestion/lineage/parser.py index 93bae226d74..b9925fb1e35 100644 --- a/ingestion/src/metadata/ingestion/lineage/parser.py +++ b/ingestion/src/metadata/ingestion/lineage/parser.py @@ -338,7 +338,7 @@ class LineageParser: logger.debug( f"Can't extract table names when parsing JOIN information from {comparison}" ) - logger.debug(f"Query: {self.masked_query}") + logger.debug(f"Query: {self.masked_query or self.query}") continue left_table_column = TableColumn(table=table_left, column=column_left) @@ -463,7 +463,7 @@ class LineageParser: self.masked_query = mask_query(self._clean_query, parser=lr_sqlparser) logger.debug( - f"Using sqlparse for lineage parsing for query: {self.masked_query}" + f"Using sqlparse for lineage parsing for query: {self.masked_query or self.query}" ) return lr_sqlparser diff --git a/ingestion/src/metadata/ingestion/lineage/sql_lineage.py b/ingestion/src/metadata/ingestion/lineage/sql_lineage.py index f7bd21265fb..577e6a87498 100644 --- a/ingestion/src/metadata/ingestion/lineage/sql_lineage.py +++ b/ingestion/src/metadata/ingestion/lineage/sql_lineage.py @@ -625,8 +625,8 @@ def get_lineage_by_query( try: lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds) - masked_query = lineage_parser.masked_query or query - logger.debug(f"Running lineage with query: {masked_query}") + masked_query = lineage_parser.masked_query + logger.debug(f"Running lineage with query: {masked_query or query}") raw_column_lineage = lineage_parser.column_lineage column_lineage.update(populate_column_lineage_map(raw_column_lineage)) @@ -697,7 +697,7 @@ def get_lineage_by_query( if not lineage_parser.query_parsing_success: query_parsing_failures.add( QueryParsingError( - query=masked_query, + query=masked_query or query, error=lineage_parser.query_parsing_failure_reason, ) ) @@ -729,8 +729,10 @@ def get_lineage_via_table_entity( try: lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds) - masked_query = lineage_parser.masked_query or query - logger.debug(f"Getting lineage via table entity using query: {masked_query}") + masked_query = lineage_parser.masked_query + logger.debug( + f"Getting lineage via table entity using query: {masked_query or query}" + ) to_table_name = table_entity.name.root for from_table_name in lineage_parser.source_tables: diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py index be2de4fbf4a..c0b217a804e 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py @@ -42,6 +42,8 @@ class OMetaQueryMixin: return str(result.hexdigest()) def _get_or_create_query(self, query: CreateQueryRequest) -> Optional[Query]: + if query.query.root is None: + return None query_hash = self._get_query_hash(query=query.query.root) query_entity = self.get_by_name(entity=Query, fqn=query_hash) if query_entity is None: diff --git a/ingestion/src/metadata/ingestion/source/database/usage_source.py b/ingestion/src/metadata/ingestion/source/database/usage_source.py index 6209d601364..65d5f263587 100644 --- a/ingestion/src/metadata/ingestion/source/database/usage_source.py +++ b/ingestion/src/metadata/ingestion/source/database/usage_source.py @@ -153,7 +153,7 @@ class UsageSource(QueryParserSource, ABC): if query: logger.debug( ( - f"###### USAGE QUERY #######\n{mask_query(query, self.dialect.value)}" + f"###### USAGE QUERY #######\n{mask_query(query, self.dialect.value) or query}" "\n##########################" ) )