yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-22 07:58:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix#7690: Use ES to build Tables FQN when processing Lineage (#9079)

Browse Source 
* Use ES to build Tables FQN when processing Lineage

* Fix pylent check

* Fix test failing

* Remove warning when calling ES with empty query

Co-authored-by: Pere Miquel Brull <peremiquelbrull@gmail.com>
This commit is contained in:
Nahuel 2022-12-01 09:56:42 +01:00 committed by GitHub
parent c1c9284ea8
commit 00375341f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 24 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ingestion/src/metadata/utils/elasticsearch.py
View File

@ -55,5 +55,4 @@ def get_entity_from_es_result(
return entity_list
return entity_list[0]
logger.warning("ES Query was empty")
return None

36
ingestion/src/metadata/utils/fqn.py
View File

@ -147,26 +147,26 @@ def _(
f"Service Name and Table Name should be informed, but got service=`{service_name}`, table=`{table_name}`"
)
if not database_name or not schema_name:
fqn_search_string = _build(
service_name, database_name or "*", schema_name or "*", table_name
)
fqn_search_string = _build(
service_name, database_name or "*", schema_name or "*", table_name
)
es_result = metadata.es_search_from_fqn(
entity_type=Table,
fqn_search_string=fqn_search_string,
)
entity: Optional[Union[Table, List[Table]]] = get_entity_from_es_result(
entity_list=es_result, fetch_multiple_entities=fetch_multiple_entities
)
if not entity:
return None
if fetch_multiple_entities:
return [str(table.fullyQualifiedName.__root__) for table in entity]
es_result = metadata.es_search_from_fqn(
entity_type=Table,
fqn_search_string=fqn_search_string,
)
entity: Optional[Union[Table, List[Table]]] = get_entity_from_es_result(
entity_list=es_result, fetch_multiple_entities=fetch_multiple_entities
)
# if entity not found in ES proceed to build FQN with database_name and schema_name
if not entity and database_name and schema_name:
fqn = _build(service_name, database_name, schema_name, table_name)
return [fqn] if fetch_multiple_entities else fqn
if entity and fetch_multiple_entities:
return [str(table.fullyQualifiedName.__root__) for table in entity]
if entity:
return str(entity.fullyQualifiedName.__root__)
fqn = _build(service_name, database_name, schema_name, table_name)
return [fqn] if fetch_multiple_entities else fqn
return None
@fqn_build_registry.add(DatabaseSchema)

9
ingestion/tests/unit/test_fqn.py
View File

@ -12,6 +12,7 @@
Test FQN build behavior
"""
from unittest import TestCase
from unittest.mock import MagicMock
import pytest
@ -90,8 +91,10 @@ class TestFqn(TestCase):
"""
Validate Table FQN building
"""
mocked_metadata = MagicMock()
mocked_metadata.es_search_from_fqn.return_value = None
table_fqn = fqn.build(
..., # metadata client not needed with all params
metadata=mocked_metadata,
entity_type=Table,
service_name="service",
database_name="db",
@ -101,7 +104,7 @@ class TestFqn(TestCase):
self.assertEqual(table_fqn, "service.db.schema.table")
table_fqn_dots = fqn.build(
..., # metadata client not needed with all params
metadata=mocked_metadata,
entity_type=Table,
service_name="service",
database_name="data.base",
@ -111,7 +114,7 @@ class TestFqn(TestCase):
self.assertEqual(table_fqn_dots, 'service."data.base".schema.table')
table_fqn_space = fqn.build(
..., # metadata client not needed with all params
metadata=mocked_metadata,
entity_type=Table,
service_name="service",
database_name="data base",