yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-26 17:34:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: ignored query with om-version and dbt (#8851)

Browse Source 
* Fix: ignored query with om-version and dbt

* Change based on comments
This commit is contained in:
Milan Bariya 2022-11-18 18:52:11 +05:30 committed by GitHub
parent 2c055641be
commit bc215515d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 30 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
ingestion/src/metadata/clients/databricks_client.py
View File

@ -25,6 +25,8 @@ from metadata.utils.helpers import datetime_to_ts
from metadata.utils.logger import ingestion_logger from metadata.utils.logger import ingestion_logger
logger = ingestion_logger() logger = ingestion_logger()
QUERY_WITH_OM_VERSION = '/* {"app": "OpenMetadata"'
QUERY_WITH_DBT = '/* {"app": "dbt"'
class DatabricksClient: class DatabricksClient:
@ -113,3 +115,10 @@ class DatabricksClient:
logger.error(exc) logger.error(exc)
return query_details return query_details
def is_query_valid(self, row) -> bool:
query_text = row.get("query_text")
return not (
query_text.startswith(QUERY_WITH_DBT)
or query_text.startswith(QUERY_WITH_OM_VERSION)
)

1
ingestion/src/metadata/ingestion/source/database/databricks_lineage.py
View File

@ -63,6 +63,7 @@ class DatabricksLineageSource(DatabricksQueryParserSource, LineageSource):
) )
for row in data: for row in data:
try: try:
if self.client.is_query_valid(row):
yield TableQuery( yield TableQuery(
query=row.get("query_text"), query=row.get("query_text"),
userName=row.get("user_name"), userName=row.get("user_name"),

1
ingestion/src/metadata/ingestion/source/database/databricks_usage.py
View File

@ -93,6 +93,7 @@ class DatabricksUsageSource(DatabricksQueryParserSource, UsageSource):
) )
for row in data: for row in data:
try: try:
if self.client.is_query_valid(row):
queries.append( queries.append(
TableQuery( TableQuery(
query=row.get("query_text"), query=row.get("query_text"),