yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-25 23:22:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

#14099: fix bigquery test connection & SP query (#14106)

Browse Source 
* Fix #14099: fix bigquery test connection & sp query

* fix unquote condition

* pyformat
This commit is contained in:
Mayur Singal 2023-11-28 18:32:46 +05:30 committed by GitHub
parent e2043a3f31
commit 49a5557e44
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ingestion/src/metadata/ingestion/source/database/bigquery/connection.py
View File

@ -13,6 +13,7 @@
Source connection handler Source connection handler
""" """
import os import os
from datetime import datetime
from functools import partial from functools import partial
from typing import Optional from typing import Optional
@ -122,7 +123,8 @@ def test_connection(
test_query, test_query,
engine=engine, engine=engine,
statement=BIGQUERY_TEST_STATEMENT.format( statement=BIGQUERY_TEST_STATEMENT.format(
region=service_connection.usageLocation region=service_connection.usageLocation,
creation_date=datetime.now().strftime("%Y-%m-%d"),
), ),
), ),
} }

2
ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py
View File

@ -321,7 +321,7 @@ class BigquerySource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource)
) )
query_result = [result.schema_description for result in query_resp.result()] query_result = [result.schema_description for result in query_resp.result()]
return query_result[0] return fqn.unquote_name(query_result[0])
except IndexError: except IndexError:
logger.debug(f"No dataset description found for {schema_name}") logger.debug(f"No dataset description found for {schema_name}")
except Exception as err: except Exception as err:

7
ingestion/src/metadata/ingestion/source/database/bigquery/queries.py
View File

@ -38,7 +38,8 @@ WHERE creation_time BETWEEN "{start_time}" AND "{end_time}"
) )
BIGQUERY_TEST_STATEMENT = textwrap.dedent( BIGQUERY_TEST_STATEMENT = textwrap.dedent(
"""SELECT query FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT limit 1""" """SELECT query FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT
where creation_time > '{creation_date}' limit 1"""
) )
@ -81,7 +82,7 @@ WITH SP_HISTORY AS (
user_email as user_name user_email as user_name
FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT
WHERE statement_type = 'SCRIPT' WHERE statement_type = 'SCRIPT'
AND start_time >= '{start_date}' AND creation_time >= '{start_date}'
AND job_type = "QUERY" AND job_type = "QUERY"
AND state = "DONE" AND state = "DONE"
AND error_result is NULL AND error_result is NULL
@ -102,7 +103,7 @@ Q_HISTORY AS (
WHERE statement_type <> 'SCRIPT' WHERE statement_type <> 'SCRIPT'
AND query NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' AND query NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
AND query NOT LIKE '/* {{"app": "dbt", %%}} */%%' AND query NOT LIKE '/* {{"app": "dbt", %%}} */%%'
AND start_time >= '{start_date}' AND creation_time >= '{start_date}'
AND job_type = "QUERY" AND job_type = "QUERY"
AND state = "DONE" AND state = "DONE"
AND error_result is NULL AND error_result is NULL

2
ingestion/src/metadata/utils/fqn.py
View File

@ -94,7 +94,7 @@ def _build(*args, quote: bool = True) -> str:
def unquote_name(name: str) -> str: def unquote_name(name: str) -> str:
return name[1:-1] if name is not None and '"' in name else name return name[1:-1] if name and name[0] == '"' and name[-1] == '"' else name
def quote_name(name: str) -> str: def quote_name(name: str) -> str: