yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-08 21:48:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #13607: BigQuery lineage ingestion fails when using GcpCredentialsPath authentication config (#13608)

Browse Source
This commit is contained in:
Iaroslav Frolikov 2023-10-23 12:12:06 +02:00 committed by GitHub
parent b6b19b83af
commit 420da29841
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 24 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ingestion/src/metadata/ingestion/source/database/bigquery/query_parser.py
View File

@ -23,7 +23,7 @@ from metadata.generated.schema.entity.services.connections.database.bigQueryConn
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.generated.schema.security.credentials.gcpValues import MultipleProjectId
from metadata.generated.schema.security.credentials.gcpValues import MultipleProjectId, GcpCredentialsValues
from metadata.ingestion.api.steps import InvalidSourceException
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.database.bigquery.helper import get_inspector_details
@ -69,6 +69,8 @@ class BigqueryQueryParserSource(QueryParserSource, ABC):
def get_engine(self):
if isinstance(
self.service_connection.credentials.gcpConfig, GcpCredentialsValues
) and isinstance(
self.service_connection.credentials.gcpConfig.projectId, MultipleProjectId
):
project_ids = deepcopy(

21
ingestion/tests/unit/topology/database/test_bigquery.py
View File

@ -21,6 +21,7 @@ from metadata.generated.schema.entity.data.table import TableType
from metadata.generated.schema.metadataIngestion.workflow import (
OpenMetadataWorkflowConfig,
)
from metadata.ingestion.source.database.bigquery.lineage import BigqueryLineageSource
from metadata.ingestion.source.database.bigquery.metadata import BigquerySource
mock_bq_config = {
@ -44,6 +45,11 @@ mock_bq_config = {
},
}
mock_credentials_path_bq_config = mock_bq_config
mock_credentials_path_bq_config["source"]["serviceConnection"]["config"]["credentials"]["gcpConfig"][
"__root__"
] = "credentials.json"
MOCK_DB_NAME = "random-project-id"
MOCK_SCHEMA_NAME = "test_omd"
@ -82,3 +88,18 @@ class BigqueryUnitTest(TestCase):
),
EXPECTED_URL,
)
class BigqueryLineageSourceTest(TestCase):
@patch("metadata.ingestion.source.database.bigquery.connection.get_connection")
@patch("metadata.ingestion.source.database.bigquery.connection.test_connection")
@patch("metadata.ingestion.ometa.ometa_api.OpenMetadata")
def __init__(self, methodName, get_connection, test_connection, OpenMetadata) -> None:
super().__init__(methodName)
self.config = OpenMetadataWorkflowConfig.parse_obj(mock_credentials_path_bq_config)
self.bq_query_parser = BigqueryLineageSource(self.config.source, OpenMetadata())
def test_get_engine_without_project_id_specified(self):
for engine in self.bq_query_parser.get_engine():
assert engine is self.bq_query_parser.engine