From 00375341f7718bc912b842746e64374f6c9e11e6 Mon Sep 17 00:00:00 2001 From: Nahuel Date: Thu, 1 Dec 2022 09:56:42 +0100 Subject: [PATCH] Fix#7690: Use ES to build Tables FQN when processing Lineage (#9079) * Use ES to build Tables FQN when processing Lineage * Fix pylent check * Fix test failing * Remove warning when calling ES with empty query Co-authored-by: Pere Miquel Brull --- ingestion/src/metadata/utils/elasticsearch.py | 1 - ingestion/src/metadata/utils/fqn.py | 36 +++++++++---------- ingestion/tests/unit/test_fqn.py | 9 +++-- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/ingestion/src/metadata/utils/elasticsearch.py b/ingestion/src/metadata/utils/elasticsearch.py index a11c54fd946..85c479bd44a 100644 --- a/ingestion/src/metadata/utils/elasticsearch.py +++ b/ingestion/src/metadata/utils/elasticsearch.py @@ -55,5 +55,4 @@ def get_entity_from_es_result( return entity_list return entity_list[0] - logger.warning("ES Query was empty") return None diff --git a/ingestion/src/metadata/utils/fqn.py b/ingestion/src/metadata/utils/fqn.py index 58b4d1afc74..c16a2dce3b9 100644 --- a/ingestion/src/metadata/utils/fqn.py +++ b/ingestion/src/metadata/utils/fqn.py @@ -147,26 +147,26 @@ def _( f"Service Name and Table Name should be informed, but got service=`{service_name}`, table=`{table_name}`" ) - if not database_name or not schema_name: + fqn_search_string = _build( + service_name, database_name or "*", schema_name or "*", table_name + ) - fqn_search_string = _build( - service_name, database_name or "*", schema_name or "*", table_name - ) - - es_result = metadata.es_search_from_fqn( - entity_type=Table, - fqn_search_string=fqn_search_string, - ) - entity: Optional[Union[Table, List[Table]]] = get_entity_from_es_result( - entity_list=es_result, fetch_multiple_entities=fetch_multiple_entities - ) - if not entity: - return None - if fetch_multiple_entities: - return [str(table.fullyQualifiedName.__root__) for table in entity] + es_result = metadata.es_search_from_fqn( + entity_type=Table, + fqn_search_string=fqn_search_string, + ) + entity: Optional[Union[Table, List[Table]]] = get_entity_from_es_result( + entity_list=es_result, fetch_multiple_entities=fetch_multiple_entities + ) + # if entity not found in ES proceed to build FQN with database_name and schema_name + if not entity and database_name and schema_name: + fqn = _build(service_name, database_name, schema_name, table_name) + return [fqn] if fetch_multiple_entities else fqn + if entity and fetch_multiple_entities: + return [str(table.fullyQualifiedName.__root__) for table in entity] + if entity: return str(entity.fullyQualifiedName.__root__) - fqn = _build(service_name, database_name, schema_name, table_name) - return [fqn] if fetch_multiple_entities else fqn + return None @fqn_build_registry.add(DatabaseSchema) diff --git a/ingestion/tests/unit/test_fqn.py b/ingestion/tests/unit/test_fqn.py index 82805b6f2ee..f5865ed541c 100644 --- a/ingestion/tests/unit/test_fqn.py +++ b/ingestion/tests/unit/test_fqn.py @@ -12,6 +12,7 @@ Test FQN build behavior """ from unittest import TestCase +from unittest.mock import MagicMock import pytest @@ -90,8 +91,10 @@ class TestFqn(TestCase): """ Validate Table FQN building """ + mocked_metadata = MagicMock() + mocked_metadata.es_search_from_fqn.return_value = None table_fqn = fqn.build( - ..., # metadata client not needed with all params + metadata=mocked_metadata, entity_type=Table, service_name="service", database_name="db", @@ -101,7 +104,7 @@ class TestFqn(TestCase): self.assertEqual(table_fqn, "service.db.schema.table") table_fqn_dots = fqn.build( - ..., # metadata client not needed with all params + metadata=mocked_metadata, entity_type=Table, service_name="service", database_name="data.base", @@ -111,7 +114,7 @@ class TestFqn(TestCase): self.assertEqual(table_fqn_dots, 'service."data.base".schema.table') table_fqn_space = fqn.build( - ..., # metadata client not needed with all params + metadata=mocked_metadata, entity_type=Table, service_name="service", database_name="data base",