MINOR - Fix clean_uri and add before pagination (#18826)

* print

* MINOR - Fix clean_uri and add before pagination

* MINOR - Fix clean_uri and add before pagination

ingestion/src/metadata/ingestion/ometa/models.py

@@ -30,3 +30,4 @@ class EntityList(BaseModel, Generic[T]):
     entities: List[T]
     total: int
     after: Optional[str] = None
+    before: Optional[str] = None

ingestion/src/metadata/ingestion/ometa/ometa_api.py

@@ -378,12 +378,13 @@ class OpenMetadata(
         logger.debug("Cannot find the Entity %s", fqn)
         return None
 
-    # pylint: disable=too-many-locals
+    # pylint: disable=too-many-locals, too-many-arguments
     def list_entities(
         self,
         entity: Type[T],
         fields: Optional[List[str]] = None,
         after: Optional[str] = None,
+        before: Optional[str] = None,
         limit: int = 100,
         params: Optional[Dict[str, str]] = None,
         skip_on_failure: bool = False,
@@ -395,9 +396,10 @@ class OpenMetadata(
         suffix = self.get_suffix(entity)
         url_limit = f"?limit={limit}"
         url_after = f"&after={after}" if after else ""
+        url_before = f"&before={before}" if before else ""
         url_fields = f"&fields={','.join(fields)}" if fields else ""
         resp = self.client.get(
-            path=f"{suffix}{url_limit}{url_after}{url_fields}", data=params
+            path=f"{suffix}{url_limit}{url_after}{url_before}{url_fields}", data=params
         )
 
         if self._use_raw_data:
@@ -421,7 +423,8 @@ class OpenMetadata(
 
         total = resp["paging"]["total"]
         after = resp["paging"]["after"] if "after" in resp["paging"] else None
-        return EntityList(entities=entities, total=total, after=after)
+        before = resp["paging"]["before"] if "before" in resp["paging"] else None
+        return EntityList(entities=entities, total=total, after=after, before=before)
 
     def list_all_entities(
         self,

ingestion/src/metadata/utils/helpers.py

@@ -360,8 +360,7 @@ def clean_uri(uri: Union[str, Url]) -> str:
     make it http://localhost:9000
     """
     # force a string of the given Uri if needed
-    if isinstance(uri, Url):
-        uri = str(uri)
+    uri = str(uri)
     return uri[:-1] if uri.endswith("/") else uri
 
 

ingestion/tests/integration/ometa/test_ometa_table_api.py

@@ -299,7 +299,7 @@ class OMetaTableTest(TestCase):
         )
         assert data
 
-    def test_list_all(self):
+    def test_list_all_and_paginate(self):
         """
         Validate generator utility to fetch all tables
         """
@@ -315,6 +315,17 @@ class OMetaTableTest(TestCase):
             len(list(all_entities)) >= 10
         )  # In case the default testing entity is not present
 
+        entity_list = self.metadata.list_entities(entity=Table, limit=2)
+        assert len(entity_list.entities) == 2
+        after_entity_list = self.metadata.list_entities(
+            entity=Table, limit=2, after=entity_list.after
+        )
+        assert len(after_entity_list.entities) == 2
+        before_entity_list = self.metadata.list_entities(
+            entity=Table, limit=2, before=after_entity_list.before
+        )
+        assert before_entity_list.entities == entity_list.entities
+
     def test_delete(self):
         """
         We can delete a Table by ID