diff --git a/ingestion/src/metadata/ingestion/ometa/models.py b/ingestion/src/metadata/ingestion/ometa/models.py index 0611d515b05..445e421e7b8 100644 --- a/ingestion/src/metadata/ingestion/ometa/models.py +++ b/ingestion/src/metadata/ingestion/ometa/models.py @@ -30,3 +30,4 @@ class EntityList(BaseModel, Generic[T]): entities: List[T] total: int after: Optional[str] = None + before: Optional[str] = None diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 0a99ab65522..1217d8950af 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -378,12 +378,13 @@ class OpenMetadata( logger.debug("Cannot find the Entity %s", fqn) return None - # pylint: disable=too-many-locals + # pylint: disable=too-many-locals, too-many-arguments def list_entities( self, entity: Type[T], fields: Optional[List[str]] = None, after: Optional[str] = None, + before: Optional[str] = None, limit: int = 100, params: Optional[Dict[str, str]] = None, skip_on_failure: bool = False, @@ -395,9 +396,10 @@ class OpenMetadata( suffix = self.get_suffix(entity) url_limit = f"?limit={limit}" url_after = f"&after={after}" if after else "" + url_before = f"&before={before}" if before else "" url_fields = f"&fields={','.join(fields)}" if fields else "" resp = self.client.get( - path=f"{suffix}{url_limit}{url_after}{url_fields}", data=params + path=f"{suffix}{url_limit}{url_after}{url_before}{url_fields}", data=params ) if self._use_raw_data: @@ -421,7 +423,8 @@ class OpenMetadata( total = resp["paging"]["total"] after = resp["paging"]["after"] if "after" in resp["paging"] else None - return EntityList(entities=entities, total=total, after=after) + before = resp["paging"]["before"] if "before" in resp["paging"] else None + return EntityList(entities=entities, total=total, after=after, before=before) def list_all_entities( self, diff --git a/ingestion/src/metadata/utils/helpers.py b/ingestion/src/metadata/utils/helpers.py index 825ab6bfd3a..0b3438c7f28 100644 --- a/ingestion/src/metadata/utils/helpers.py +++ b/ingestion/src/metadata/utils/helpers.py @@ -360,8 +360,7 @@ def clean_uri(uri: Union[str, Url]) -> str: make it http://localhost:9000 """ # force a string of the given Uri if needed - if isinstance(uri, Url): - uri = str(uri) + uri = str(uri) return uri[:-1] if uri.endswith("/") else uri diff --git a/ingestion/tests/integration/ometa/test_ometa_table_api.py b/ingestion/tests/integration/ometa/test_ometa_table_api.py index 8845dc8efa3..f8e673b5dcc 100644 --- a/ingestion/tests/integration/ometa/test_ometa_table_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_table_api.py @@ -299,7 +299,7 @@ class OMetaTableTest(TestCase): ) assert data - def test_list_all(self): + def test_list_all_and_paginate(self): """ Validate generator utility to fetch all tables """ @@ -315,6 +315,17 @@ class OMetaTableTest(TestCase): len(list(all_entities)) >= 10 ) # In case the default testing entity is not present + entity_list = self.metadata.list_entities(entity=Table, limit=2) + assert len(entity_list.entities) == 2 + after_entity_list = self.metadata.list_entities( + entity=Table, limit=2, after=entity_list.after + ) + assert len(after_entity_list.entities) == 2 + before_entity_list = self.metadata.list_entities( + entity=Table, limit=2, before=after_entity_list.before + ) + assert before_entity_list.entities == entity_list.entities + def test_delete(self): """ We can delete a Table by ID