diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java index 2eb6e4f043f..55f29b57a04 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java @@ -526,10 +526,13 @@ class DashboardESIndex extends ElasticSearchIndex { dashboard.getTags().forEach(tag -> tags.add(tag.getTagFQN())); } - for (EntityReference chart : dashboard.getCharts()) { - chartNames.add(chart.getDisplayName()); - chartDescriptions.add(chart.getDescription()); + if (dashboard.getCharts() != null) { + for (EntityReference chart : dashboard.getCharts()) { + chartNames.add(chart.getDisplayName()); + chartDescriptions.add(chart.getDescription()); + } } + ParseTags parseTags = new ParseTags(tags); String description = dashboard.getDescription() != null ? dashboard.getDescription() : ""; String displayName = dashboard.getDisplayName() != null ? dashboard.getDisplayName() : ""; diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/es_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/es_mixin.py index 1f18ca04a80..7e4ac2ec191 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/es_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/es_mixin.py @@ -80,7 +80,7 @@ class ESMixin(Generic[T]): :param retries: Times to retry :return: List of Entities or None """ - times = min(1, retries) # Try at least once + times = max(1, retries) # Try at least once while times: entity_list = self._search_es_entity( entity_type=entity_type, query_string=query_string diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest.py b/ingestion/src/metadata/ingestion/sink/metadata_rest.py index 363a7a20d3e..50bc8acd42e 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest.py @@ -582,6 +582,7 @@ class MetadataRestSink(Sink[Entity]): if "." not in from_table_name: from_table_name = f"{db_schema.name.__root__}.{from_table_name}" _create_lineage_by_table_name( + self.metadata, from_table_name, f"{db_schema.name.__root__}.{to_table_name}", db.service.name, diff --git a/ingestion/src/metadata/utils/fqn.py b/ingestion/src/metadata/utils/fqn.py index 3e99d84eeab..1b1576f6f32 100644 --- a/ingestion/src/metadata/utils/fqn.py +++ b/ingestion/src/metadata/utils/fqn.py @@ -119,6 +119,7 @@ def _( database_name: Optional[str], schema_name: Optional[str], table_name: str, + retries: int = 3, ) -> Optional[str]: """ Building logic for tables @@ -127,6 +128,7 @@ def _( :param database_name: DB name or None :param schema_name: Schema name or None :param table_name: Table name + :param retries: ES Search retries :return: """ if not service_name or not table_name: @@ -143,6 +145,7 @@ def _( "database_schema": schema_name, "name": table_name, }, + retries=retries, ) entity: Optional[Table] = get_entity_from_es_result(entity_list=es_result) return str(entity.fullyQualifiedName.__root__) if entity else None diff --git a/ingestion/tests/integration/ometa/test_ometa_es_api.py b/ingestion/tests/integration/ometa/test_ometa_es_api.py index 72dc6986711..e7b2d58d5e9 100644 --- a/ingestion/tests/integration/ometa/test_ometa_es_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_es_api.py @@ -49,11 +49,6 @@ class OMetaESTest(TestCase): assert metadata.health_check() - user = metadata.create_or_update( - data=CreateUserRequest(name="random-user", email="random@user.com"), - ) - owner = EntityReference(id=user.id, type="user") - service = CreateDatabaseServiceRequest( name="test-service-es", serviceType=DatabaseServiceType.Mysql, @@ -76,28 +71,28 @@ class OMetaESTest(TestCase): cls.service_entity = cls.metadata.create_or_update(data=cls.service) create_db = CreateDatabaseRequest( - name="test-db", + name="test-db-es", service=EntityReference(id=cls.service_entity.id, type="databaseService"), ) create_db_entity = cls.metadata.create_or_update(data=create_db) cls.db_reference = EntityReference( - id=create_db_entity.id, name="test-db", type="database" + id=create_db_entity.id, name="test-db-es", type="database" ) create_schema = CreateDatabaseSchemaRequest( - name="test-schema", database=cls.db_reference + name="test-schema-es", database=cls.db_reference ) create_schema_entity = cls.metadata.create_or_update(data=create_schema) cls.schema_reference = EntityReference( - id=create_schema_entity.id, name="test-schema", type="databaseSchema" + id=create_schema_entity.id, name="test-schema-es", type="databaseSchema" ) create = CreateTableRequest( - name="test", + name="test-es", databaseSchema=cls.schema_reference, columns=[Column(name="id", dataType=DataType.BIGINT)], ) @@ -131,6 +126,8 @@ class OMetaESTest(TestCase): entity_type=Table, service_name=self.service.name.__root__, filters={"name": self.entity.name.__root__}, + size=100, + retries=10, ) # We get the created table back @@ -144,6 +141,8 @@ class OMetaESTest(TestCase): "name": self.entity.name.__root__, "database": self.db_reference.name, }, + size=100, + retries=10, ) self.assertIsNotNone(res) @@ -157,6 +156,8 @@ class OMetaESTest(TestCase): "database": self.db_reference.name, "database_schema": self.schema_reference.name, }, + size=100, + retries=10, ) self.assertIsNotNone(res) diff --git a/ingestion/tests/integration/utils/test_fqn.py b/ingestion/tests/integration/utils/test_fqn.py index ef1bb2b0ef4..cf5b7ec0ab3 100644 --- a/ingestion/tests/integration/utils/test_fqn.py +++ b/ingestion/tests/integration/utils/test_fqn.py @@ -143,6 +143,7 @@ class FQNBuildTest(TestCase): database_name=self.db_reference.name, schema_name=None, table_name=self.entity.name.__root__, + retries=5, ) self.assertEqual("test-service-table-fqn.test-db.test-schema.test", table_fqn) @@ -154,6 +155,7 @@ class FQNBuildTest(TestCase): database_name=None, schema_name=None, table_name=self.entity.name.__root__, + retries=5, ) self.assertEqual("test-service-table-fqn.test-db.test-schema.test", table_fqn) diff --git a/ingestion/tests/unit/test_fqn.py b/ingestion/tests/unit/test_fqn.py index e7a2a7139ee..a595b45fc0e 100644 --- a/ingestion/tests/unit/test_fqn.py +++ b/ingestion/tests/unit/test_fqn.py @@ -89,6 +89,7 @@ class TestFqn(TestCase): Validate Table FQN building """ table_fqn = fqn.build( + ..., # metadata client not needed with all params entity_type=Table, service_name="service", database_name="db", @@ -98,6 +99,7 @@ class TestFqn(TestCase): self.assertEqual(table_fqn, "service.db.schema.table") table_fqn_dots = fqn.build( + ..., # metadata client not needed with all params entity_type=Table, service_name="service", database_name="data.base", @@ -107,6 +109,7 @@ class TestFqn(TestCase): self.assertEqual(table_fqn_dots, 'service."data.base".schema.table') table_fqn_space = fqn.build( + ..., # metadata client not needed with all params entity_type=Table, service_name="service", database_name="data base",