yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-22 16:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix#7690: Use ES to build Tables FQN when processing Lineage (#9079)

Browse Source 
* Use ES to build Tables FQN when processing Lineage

* Fix pylent check

* Fix test failing

* Remove warning when calling ES with empty query

Co-authored-by: Pere Miquel Brull <peremiquelbrull@gmail.com>
This commit is contained in:
Nahuel 2022-12-01 09:56:42 +01:00 committed by GitHub
parent c1c9284ea8
commit 00375341f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 24 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ingestion/src/metadata/utils/elasticsearch.py
View File

@ -55,5 +55,4 @@ def get_entity_from_es_result(
return entity_list return entity_list
return entity_list[0] return entity_list[0]
logger.warning("ES Query was empty")
return None return None

36
ingestion/src/metadata/utils/fqn.py
View File

@ -147,26 +147,26 @@ def _(
f"Service Name and Table Name should be informed, but got service=`{service_name}`, table=`{table_name}`" f"Service Name and Table Name should be informed, but got service=`{service_name}`, table=`{table_name}`"
) )
if not database_name or not schema_name: fqn_search_string = _build(
service_name, database_name or "*", schema_name or "*", table_name
)
fqn_search_string = _build( es_result = metadata.es_search_from_fqn(
service_name, database_name or "*", schema_name or "*", table_name entity_type=Table,
) fqn_search_string=fqn_search_string,
)
es_result = metadata.es_search_from_fqn( entity: Optional[Union[Table, List[Table]]] = get_entity_from_es_result(
entity_type=Table, entity_list=es_result, fetch_multiple_entities=fetch_multiple_entities
fqn_search_string=fqn_search_string, )
) # if entity not found in ES proceed to build FQN with database_name and schema_name
entity: Optional[Union[Table, List[Table]]] = get_entity_from_es_result( if not entity and database_name and schema_name:
entity_list=es_result, fetch_multiple_entities=fetch_multiple_entities fqn = _build(service_name, database_name, schema_name, table_name)
) return [fqn] if fetch_multiple_entities else fqn
if not entity: if entity and fetch_multiple_entities:
return None return [str(table.fullyQualifiedName.__root__) for table in entity]
if fetch_multiple_entities: if entity:
return [str(table.fullyQualifiedName.__root__) for table in entity]
return str(entity.fullyQualifiedName.__root__) return str(entity.fullyQualifiedName.__root__)
fqn = _build(service_name, database_name, schema_name, table_name) return None
return [fqn] if fetch_multiple_entities else fqn
@fqn_build_registry.add(DatabaseSchema) @fqn_build_registry.add(DatabaseSchema)

9
ingestion/tests/unit/test_fqn.py
View File

@ -12,6 +12,7 @@
Test FQN build behavior Test FQN build behavior
""" """
from unittest import TestCase from unittest import TestCase
from unittest.mock import MagicMock
import pytest import pytest
@ -90,8 +91,10 @@ class TestFqn(TestCase):
""" """
Validate Table FQN building Validate Table FQN building
""" """
mocked_metadata = MagicMock()
mocked_metadata.es_search_from_fqn.return_value = None
table_fqn = fqn.build( table_fqn = fqn.build(
..., # metadata client not needed with all params metadata=mocked_metadata,
entity_type=Table, entity_type=Table,
service_name="service", service_name="service",
database_name="db", database_name="db",
@ -101,7 +104,7 @@ class TestFqn(TestCase):
self.assertEqual(table_fqn, "service.db.schema.table") self.assertEqual(table_fqn, "service.db.schema.table")
table_fqn_dots = fqn.build( table_fqn_dots = fqn.build(
..., # metadata client not needed with all params metadata=mocked_metadata,
entity_type=Table, entity_type=Table,
service_name="service", service_name="service",
database_name="data.base", database_name="data.base",
@ -111,7 +114,7 @@ class TestFqn(TestCase):
self.assertEqual(table_fqn_dots, 'service."data.base".schema.table') self.assertEqual(table_fqn_dots, 'service."data.base".schema.table')
table_fqn_space = fqn.build( table_fqn_space = fqn.build(
..., # metadata client not needed with all params metadata=mocked_metadata,
entity_type=Table, entity_type=Table,
service_name="service", service_name="service",
database_name="data base", database_name="data base",