From 2888e379984d1342f00b86c81ff15dca2cb0c297 Mon Sep 17 00:00:00 2001 From: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:55:12 +0530 Subject: [PATCH] Search Index read entities from index mapping (#19084) --- .../bundles/searchIndex/SearchIndexApp.java | 89 +------------------ .../service/search/SearchRepository.java | 5 ++ .../SearchIndexingApplication.json | 62 ++++++------- 3 files changed, 40 insertions(+), 116 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index fd34c2027f5..1bb8cb28361 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -1,47 +1,8 @@ package org.openmetadata.service.apps.bundles.searchIndex; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.service.Entity.API_COLLCECTION; -import static org.openmetadata.service.Entity.API_ENDPOINT; -import static org.openmetadata.service.Entity.API_SERVICE; -import static org.openmetadata.service.Entity.CHART; -import static org.openmetadata.service.Entity.CLASSIFICATION; -import static org.openmetadata.service.Entity.CONTAINER; -import static org.openmetadata.service.Entity.DASHBOARD; -import static org.openmetadata.service.Entity.DASHBOARD_DATA_MODEL; -import static org.openmetadata.service.Entity.DASHBOARD_SERVICE; -import static org.openmetadata.service.Entity.DATABASE; -import static org.openmetadata.service.Entity.DATABASE_SCHEMA; -import static org.openmetadata.service.Entity.DATABASE_SERVICE; -import static org.openmetadata.service.Entity.DATA_PRODUCT; -import static org.openmetadata.service.Entity.DOMAIN; -import static org.openmetadata.service.Entity.ENTITY_REPORT_DATA; -import static org.openmetadata.service.Entity.GLOSSARY; -import static org.openmetadata.service.Entity.GLOSSARY_TERM; -import static org.openmetadata.service.Entity.INGESTION_PIPELINE; -import static org.openmetadata.service.Entity.MESSAGING_SERVICE; -import static org.openmetadata.service.Entity.METADATA_SERVICE; -import static org.openmetadata.service.Entity.METRIC; -import static org.openmetadata.service.Entity.MLMODEL; -import static org.openmetadata.service.Entity.MLMODEL_SERVICE; -import static org.openmetadata.service.Entity.PIPELINE; -import static org.openmetadata.service.Entity.PIPELINE_SERVICE; -import static org.openmetadata.service.Entity.QUERY; -import static org.openmetadata.service.Entity.SEARCH_INDEX; -import static org.openmetadata.service.Entity.SEARCH_SERVICE; -import static org.openmetadata.service.Entity.STORAGE_SERVICE; -import static org.openmetadata.service.Entity.STORED_PROCEDURE; -import static org.openmetadata.service.Entity.TABLE; -import static org.openmetadata.service.Entity.TAG; -import static org.openmetadata.service.Entity.TEAM; -import static org.openmetadata.service.Entity.TEST_CASE; import static org.openmetadata.service.Entity.TEST_CASE_RESOLUTION_STATUS; import static org.openmetadata.service.Entity.TEST_CASE_RESULT; -import static org.openmetadata.service.Entity.TEST_SUITE; -import static org.openmetadata.service.Entity.TOPIC; -import static org.openmetadata.service.Entity.USER; -import static org.openmetadata.service.Entity.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA; -import static org.openmetadata.service.Entity.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA; import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS; import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.WEBSOCKET_STATUS_CHANNEL; import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB; @@ -99,53 +60,8 @@ import org.quartz.JobExecutionContext; @Slf4j public class SearchIndexApp extends AbstractNativeApplication { - private static final String ALL = "all"; - public static final Set ALL_ENTITIES = - Set.of( - TABLE, - DASHBOARD, - TOPIC, - PIPELINE, - INGESTION_PIPELINE, - SEARCH_INDEX, - USER, - TEAM, - GLOSSARY, - GLOSSARY_TERM, - MLMODEL, - TAG, - CLASSIFICATION, - QUERY, - CONTAINER, - DATABASE, - DATABASE_SCHEMA, - TEST_CASE, - TEST_SUITE, - CHART, - DASHBOARD_DATA_MODEL, - DATABASE_SERVICE, - MESSAGING_SERVICE, - DASHBOARD_SERVICE, - PIPELINE_SERVICE, - MLMODEL_SERVICE, - STORAGE_SERVICE, - METADATA_SERVICE, - SEARCH_SERVICE, - ENTITY_REPORT_DATA, - WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA, - WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA, - DOMAIN, - STORED_PROCEDURE, - DATA_PRODUCT, - TEST_CASE_RESOLUTION_STATUS, - TEST_CASE_RESULT, - API_SERVICE, - API_ENDPOINT, - API_COLLCECTION, - METRIC); - public static final Set TIME_SERIES_ENTITIES = Set.of( ReportData.ReportDataType.ENTITY_REPORT_DATA.value(), @@ -180,8 +96,9 @@ public class SearchIndexApp extends AbstractNativeApplication { JsonUtils.convertValue(app.getAppConfiguration(), EventPublisherJob.class) .withStats(new Stats()); - if (request.getEntities().contains(ALL)) { - request.setEntities(ALL_ENTITIES); + if (request.getEntities().size() == 1 && request.getEntities().contains(ALL)) { + SearchRepository searchRepo = Entity.getSearchRepo(); + request.setEntities(searchRepo.getSearchEntities()); } jobData = request; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 98f4477afdf..d1e368fdc46 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -1057,4 +1058,8 @@ public class SearchRepository { } return new ArrayList<>(); } + + public Set getSearchEntities() { + return new HashSet<>(entityIndexMap.keySet()); + } } diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json index 4957d1979fa..ff25b2618b6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json @@ -66,46 +66,48 @@ "type": "string", "enum": [ "table", - "dashboard", - "topic", - "pipeline", - "ingestionPipeline", - "searchIndex", - "user", - "team", - "glossary", - "glossaryTerm", - "mlmodel", - "tag", - "classification", - "query", - "container", - "database", "databaseSchema", - "testCase", + "container", "testSuite", - "chart", - "dashboardDataModel", - "databaseService", - "messagingService", - "dashboardService", - "pipelineService", "mlmodelService", - "storageService", + "pipelineService", + "messagingService", + "entityReportData", + "ingestionPipeline", + "database", "metadataService", "searchService", - "entityReportData", + "aggregatedCostAnalysisReportData", + "tag", + "dashboard", + "rawCostAnalysisReportData", "webAnalyticEntityViewReportData", - "webAnalyticUserActivityReportData", - "domain", "storedProcedure", "dataProduct", - "testCaseResolutionStatus", - "testCaseResult", + "databaseService", + "dashboardService", + "query", "apiService", - "apiEndpoint", + "searchIndex", + "testCaseResult", "apiCollection", - "metric" + "team", + "mlmodel", + "classification", + "glossaryTerm", + "testCaseResolutionStatus", + "dashboardDataModel", + "pipeline", + "glossary", + "apiEndpoint", + "storageService", + "metric", + "webAnalyticUserActivityReportData", + "domain", + "topic", + "chart", + "user", + "testCase" ] }, "default": ["all"],