diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/metadataESConnection.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/metadataESConnection.json index 6d52611dae8..2af1d86ef9b 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/metadataESConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/metadata/metadataESConnection.json @@ -54,6 +54,31 @@ "type": "boolean", "default": "true" }, + "includePolicy": { + "description": "Include Tags for Policy", + "type": "boolean", + "default": true + }, + "includeMessagingServices": { + "description": "Include Messaging Services for Indexing", + "type": "boolean", + "default": true + }, + "includeDatabaseServices": { + "description": "Include Database Services for Indexing", + "type": "boolean", + "default": true + }, + "includePipelineServices": { + "description": "Include Pipeline Services for Indexing", + "type": "boolean", + "default": true + }, + "includeTags": { + "description": "Include Tags for Indexing", + "type": "boolean", + "default": true + }, "limitRecords": { "description": "Limit the number of records for Indexing.", "type": "integer", diff --git a/ingestion/src/metadata/ingestion/source/metadata.py b/ingestion/src/metadata/ingestion/source/metadata.py index d424f7ff8b4..d67dc4549f9 100644 --- a/ingestion/src/metadata/ingestion/source/metadata.py +++ b/ingestion/src/metadata/ingestion/source/metadata.py @@ -26,7 +26,7 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.services.messagingService import MessagingService from metadata.generated.schema.entity.services.pipelineService import PipelineService -from metadata.generated.schema.entity.tags.tagCategory import Tag +from metadata.generated.schema.entity.tags.tagCategory import TagCategory from metadata.generated.schema.entity.teams.team import Team from metadata.generated.schema.entity.teams.user import User from metadata.generated.schema.metadataIngestion.workflow import ( @@ -183,7 +183,7 @@ class MetadataSource(Source[Entity]): ) if self.service_connection.includeTags: yield from self.fetch_entities( - entity_class=Tag, + entity_class=TagCategory, fields=[], ) @@ -206,20 +206,25 @@ class MetadataSource(Source[Entity]): ) def fetch_entities(self, entity_class, fields): - after = None - while True: - entities_list = self.metadata.list_entities( - entity=entity_class, - fields=fields, - after=after, - limit=self.service_connection.limitRecords, - ) - for entity in entities_list.entities: - self.status.scanned_entity(entity_class.__name__, entity.name) - yield entity - if entities_list.after is None: - break - after = entities_list.after + try: + after = None + while True: + entities_list = self.metadata.list_entities( + entity=entity_class, + fields=fields, + after=after, + limit=self.service_connection.limitRecords, + ) + for entity in entities_list.entities: + self.status.scanned_entity(entity_class.__name__, entity.name) + yield entity + if entities_list.after is None: + break + after = entities_list.after + + except Exception as err: + logger.debug(err) + logger.error(f"Fetching entities failed for {entity_class.__name__}") def get_status(self) -> SourceStatus: return self.status