diff --git a/ingestion/src/metadata/ingestion/source/search/elasticsearch/metadata.py b/ingestion/src/metadata/ingestion/source/search/elasticsearch/metadata.py index 191662523b7..1d2214656c4 100644 --- a/ingestion/src/metadata/ingestion/source/search/elasticsearch/metadata.py +++ b/ingestion/src/metadata/ingestion/source/search/elasticsearch/metadata.py @@ -12,6 +12,7 @@ Elasticsearch source to extract metadata """ import shutil +import traceback from pathlib import Path from typing import Any, Iterable, Optional @@ -21,6 +22,7 @@ from metadata.generated.schema.api.data.createSearchIndex import ( CreateSearchIndexRequest, ) from metadata.generated.schema.entity.data.searchIndex import ( + IndexType, SearchIndex, SearchIndexSampleData, ) @@ -103,6 +105,7 @@ class ElasticsearchSource(SearchServiceSource): fields=parse_es_index_mapping( search_index_details.get(index_name, {}).get("mappings") ), + indexType=IndexType.Index, ) yield Either(right=search_index_request) self.register_record(search_index_request=search_index_request) @@ -143,6 +146,56 @@ class ElasticsearchSource(SearchServiceSource): ) ) + def get_search_index_template_list(self) -> Iterable[dict]: + """ + Get List of all search index template + """ + yield from self.client.indices.get_index_template().get("index_templates", []) + + def get_search_index_template_name( + self, search_index_template_details: dict + ) -> Optional[str]: + """ + Get Search Index Template Name + """ + return search_index_template_details and search_index_template_details["name"] + + def yield_search_index_template( + self, search_index_template_details: Any + ) -> Iterable[Either[CreateSearchIndexRequest]]: + """ + Method to Get Search Index Template Entity + """ + try: + if self.source_config.includeIndexTemplate: + index_name = self.get_search_index_template_name( + search_index_template_details + ) + index_template = search_index_template_details["index_template"] + if index_name: + search_index_template_request = CreateSearchIndexRequest( + name=EntityName(index_name), + displayName=index_name, + searchIndexSettings=index_template.get("template", {}).get( + "settings", {} + ), + service=FullyQualifiedEntityName( + self.context.get().search_service + ), + fields=parse_es_index_mapping( + index_template.get("template", {}).get("mappings") + ), + indexType=IndexType.IndexTemplate, + description=index_template.get("_meta", {}).get("description"), + ) + yield Either(right=search_index_template_request) + self.register_record( + search_index_request=search_index_template_request + ) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.error(f"Could not include index templates due to {exc}") + def close(self): try: if Path(self.service_connection.sslConfig.certificates.stagingDir).exists(): diff --git a/ingestion/src/metadata/ingestion/source/search/search_service.py b/ingestion/src/metadata/ingestion/source/search/search_service.py index 6bf2eaadd37..8ef0400748a 100644 --- a/ingestion/src/metadata/ingestion/source/search/search_service.py +++ b/ingestion/src/metadata/ingestion/source/search/search_service.py @@ -83,7 +83,7 @@ class SearchServiceTopology(ServiceTopology): cache_entities=True, ), ], - children=["search_index"], + children=["search_index", "search_index_template"], post_process=["mark_search_indexes_as_deleted"], ) search_index: Annotated[ @@ -107,6 +107,21 @@ class SearchServiceTopology(ServiceTopology): ], ) + search_index_template: Annotated[ + TopologyNode, Field(description="Search Index Template Processing Node") + ] = TopologyNode( + producer="get_search_index_template", + stages=[ + NodeStage( + type_=SearchIndex, + context="search_index_template", + processor="yield_search_index_template", + consumer=["search_service"], + use_cache=True, + ) + ], + ) + class SearchServiceSource(TopologyRunnerMixin, Source, ABC): """ @@ -178,6 +193,34 @@ class SearchServiceSource(TopologyRunnerMixin, Source, ABC): continue yield index_details + def yield_search_index_template( + self, search_index_template_details: Any + ) -> Iterable[Either[CreateSearchIndexRequest]]: + """Method to Get Search Index Templates""" + + def get_search_index_template_list(self) -> Optional[List[Any]]: + """Get list of all search index templates""" + + def get_search_index_template_name(self, search_index_template_details: Any) -> str: + """Get Search Index Template Name""" + + def get_search_index_template(self) -> Any: + if self.source_config.includeIndexTemplate: + for index_template_details in self.get_search_index_template_list(): + if search_index_template_name := self.get_search_index_template_name( + index_template_details + ): + if filter_by_search_index( + self.source_config.searchIndexFilterPattern, + search_index_template_name, + ): + self.status.filter( + search_index_template_name, + "Search Index Template Filtered Out", + ) + continue + yield index_template_details + def yield_create_request_search_service( self, config: WorkflowSource ) -> Iterable[Either[CreateSearchServiceRequest]]: diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java index e20a5d37c24..4bc11613342 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java @@ -389,6 +389,7 @@ public class SearchIndexRepository extends EntityRepository { original.getSearchIndexSettings(), updated.getSearchIndexSettings()); recordChange("sourceHash", original.getSourceHash(), updated.getSourceHash()); + recordChange("indexType", original.getIndexType(), updated.getIndexType()); } private void updateSearchIndexFields( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java index 628e34670d8..24831f21253 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java @@ -628,11 +628,14 @@ public class SearchIndexResource extends EntityResource