Fix Python tests & flakiness (#5095)

Fix Python tests & flakiness (#5095)

catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchIndexDefinition.java

@@ -526,10 +526,13 @@ class DashboardESIndex extends ElasticSearchIndex {
       dashboard.getTags().forEach(tag -> tags.add(tag.getTagFQN()));
     }
 
-    for (EntityReference chart : dashboard.getCharts()) {
+    if (dashboard.getCharts() != null) {
-      chartNames.add(chart.getDisplayName());
+      for (EntityReference chart : dashboard.getCharts()) {
-      chartDescriptions.add(chart.getDescription());
+        chartNames.add(chart.getDisplayName());
+        chartDescriptions.add(chart.getDescription());
+      }
     }
+
     ParseTags parseTags = new ParseTags(tags);
     String description = dashboard.getDescription() != null ? dashboard.getDescription() : "";
     String displayName = dashboard.getDisplayName() != null ? dashboard.getDisplayName() : "";

ingestion/src/metadata/ingestion/ometa/mixins/es_mixin.py

@@ -80,7 +80,7 @@ class ESMixin(Generic[T]):
         :param retries: Times to retry
         :return: List of Entities or None
         """
-        times = min(1, retries)  # Try at least once
+        times = max(1, retries)  # Try at least once
         while times:
             entity_list = self._search_es_entity(
                 entity_type=entity_type, query_string=query_string

ingestion/src/metadata/ingestion/sink/metadata_rest.py

@@ -582,6 +582,7 @@ class MetadataRestSink(Sink[Entity]):
                 if "." not in from_table_name:
                     from_table_name = f"{db_schema.name.__root__}.{from_table_name}"
                 _create_lineage_by_table_name(
+                    self.metadata,
                     from_table_name,
                     f"{db_schema.name.__root__}.{to_table_name}",
                     db.service.name,

ingestion/src/metadata/utils/fqn.py

@@ -119,6 +119,7 @@ def _(
     database_name: Optional[str],
     schema_name: Optional[str],
     table_name: str,
+    retries: int = 3,
 ) -> Optional[str]:
     """
     Building logic for tables
@@ -127,6 +128,7 @@ def _(
     :param database_name: DB name or None
     :param schema_name: Schema name or None
     :param table_name: Table name
+    :param retries: ES Search retries
     :return:
     """
     if not service_name or not table_name:
@@ -143,6 +145,7 @@ def _(
                 "database_schema": schema_name,
                 "name": table_name,
             },
+            retries=retries,
         )
         entity: Optional[Table] = get_entity_from_es_result(entity_list=es_result)
         return str(entity.fullyQualifiedName.__root__) if entity else None

ingestion/tests/integration/ometa/test_ometa_es_api.py

@@ -49,11 +49,6 @@ class OMetaESTest(TestCase):
 
     assert metadata.health_check()
 
-    user = metadata.create_or_update(
-        data=CreateUserRequest(name="random-user", email="random@user.com"),
-    )
-    owner = EntityReference(id=user.id, type="user")
-
     service = CreateDatabaseServiceRequest(
         name="test-service-es",
         serviceType=DatabaseServiceType.Mysql,
@@ -76,28 +71,28 @@ class OMetaESTest(TestCase):
         cls.service_entity = cls.metadata.create_or_update(data=cls.service)
 
         create_db = CreateDatabaseRequest(
-            name="test-db",
+            name="test-db-es",
             service=EntityReference(id=cls.service_entity.id, type="databaseService"),
         )
 
         create_db_entity = cls.metadata.create_or_update(data=create_db)
 
         cls.db_reference = EntityReference(
-            id=create_db_entity.id, name="test-db", type="database"
+            id=create_db_entity.id, name="test-db-es", type="database"
         )
 
         create_schema = CreateDatabaseSchemaRequest(
-            name="test-schema", database=cls.db_reference
+            name="test-schema-es", database=cls.db_reference
         )
 
         create_schema_entity = cls.metadata.create_or_update(data=create_schema)
 
         cls.schema_reference = EntityReference(
-            id=create_schema_entity.id, name="test-schema", type="databaseSchema"
+            id=create_schema_entity.id, name="test-schema-es", type="databaseSchema"
         )
 
         create = CreateTableRequest(
-            name="test",
+            name="test-es",
             databaseSchema=cls.schema_reference,
             columns=[Column(name="id", dataType=DataType.BIGINT)],
         )
@@ -131,6 +126,8 @@ class OMetaESTest(TestCase):
             entity_type=Table,
             service_name=self.service.name.__root__,
             filters={"name": self.entity.name.__root__},
+            size=100,
+            retries=10,
         )
 
         # We get the created table back
@@ -144,6 +141,8 @@ class OMetaESTest(TestCase):
                 "name": self.entity.name.__root__,
                 "database": self.db_reference.name,
             },
+            size=100,
+            retries=10,
         )
 
         self.assertIsNotNone(res)
@@ -157,6 +156,8 @@ class OMetaESTest(TestCase):
                 "database": self.db_reference.name,
                 "database_schema": self.schema_reference.name,
             },
+            size=100,
+            retries=10,
         )
 
         self.assertIsNotNone(res)

ingestion/tests/integration/utils/test_fqn.py

@@ -143,6 +143,7 @@ class FQNBuildTest(TestCase):
             database_name=self.db_reference.name,
             schema_name=None,
             table_name=self.entity.name.__root__,
+            retries=5,
         )
 
         self.assertEqual("test-service-table-fqn.test-db.test-schema.test", table_fqn)
@@ -154,6 +155,7 @@ class FQNBuildTest(TestCase):
             database_name=None,
             schema_name=None,
             table_name=self.entity.name.__root__,
+            retries=5,
         )
 
         self.assertEqual("test-service-table-fqn.test-db.test-schema.test", table_fqn)

ingestion/tests/unit/test_fqn.py

@@ -89,6 +89,7 @@ class TestFqn(TestCase):
         Validate Table FQN building
         """
         table_fqn = fqn.build(
+            ...,  # metadata client not needed with all params
             entity_type=Table,
             service_name="service",
             database_name="db",
@@ -98,6 +99,7 @@ class TestFqn(TestCase):
         self.assertEqual(table_fqn, "service.db.schema.table")
 
         table_fqn_dots = fqn.build(
+            ...,  # metadata client not needed with all params
             entity_type=Table,
             service_name="service",
             database_name="data.base",
@@ -107,6 +109,7 @@ class TestFqn(TestCase):
         self.assertEqual(table_fqn_dots, 'service."data.base".schema.table')
 
         table_fqn_space = fqn.build(
+            ...,  # metadata client not needed with all params
             entity_type=Table,
             service_name="service",
             database_name="data base",