From df1afbf86cf0cf73e6115e90a91e9e5cdcbaba52 Mon Sep 17 00:00:00 2001
From: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com>
Date: Mon, 8 Sep 2025 17:59:14 +0530
Subject: [PATCH] Drop Insights Indexes (#23294)

(cherry picked from commit 450840eb8e9c65e8f8224221e2f66356ed4c6d82)
---
 .../bundles/insights/DataInsightsApp.java     |  4 +--
 .../service/util/OpenMetadataOperations.java  | 29 +++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java
index 23b8a1651a8..97a3d714a7e 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java
@@ -160,7 +160,7 @@ public class DataInsightsApp extends AbstractNativeApplication {
     }
   }
 
-  private void deleteDataQualityDataIndex() {
+  public void deleteDataQualityDataIndex() {
     deleteIndexInternal(Entity.TEST_CASE_RESULT);
     deleteIndexInternal(Entity.TEST_CASE_RESOLUTION_STATUS);
   }
@@ -193,7 +193,7 @@ public class DataInsightsApp extends AbstractNativeApplication {
     }
   }
 
-  private void deleteDataAssetsDataStream() {
+  public void deleteDataAssetsDataStream() {
     DataInsightsSearchInterface searchInterface = getSearchInterface();
 
     try {
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java
index 772a7b9ce0d..a4c09f865a3 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java
@@ -71,6 +71,7 @@ import org.openmetadata.service.Entity;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
 import org.openmetadata.service.TypeRegistry;
 import org.openmetadata.service.apps.ApplicationHandler;
+import org.openmetadata.service.apps.bundles.insights.DataInsightsApp;
 import org.openmetadata.service.apps.bundles.searchIndex.SlackWebApiClient;
 import org.openmetadata.service.apps.scheduler.AppScheduler;
 import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory;
@@ -1325,10 +1326,16 @@ public class OpenMetadataOperations implements Callable<Integer> {
     try {
       LOG.info("Dropping all indexes from search engine...");
       parseConfig();
+
+      // Drop regular search repository indexes
       for (String entityType : searchRepository.getEntityIndexMap().keySet()) {
         LOG.info("Dropping index for entity type: {}", entityType);
         searchRepository.deleteIndex(searchRepository.getIndexMapping(entityType));
       }
+
+      // Drop data streams and data quality indexes created by DataInsightsApp
+      dropDataInsightsIndexes();
+
       LOG.info("All indexes dropped successfully.");
       return 0;
     } catch (Exception e) {
@@ -1337,6 +1344,28 @@ public class OpenMetadataOperations implements Callable<Integer> {
     }
   }
 
+  private void dropDataInsightsIndexes() {
+    try {
+      LOG.info("Dropping Data Insights data streams and indexes...");
+
+      // Create a DataInsightsApp instance to access its cleanup methods
+      DataInsightsApp dataInsightsApp = new DataInsightsApp(collectionDAO, searchRepository);
+
+      // Drop data assets data streams
+      LOG.info("Dropping data assets data streams...");
+      dataInsightsApp.deleteDataAssetsDataStream();
+
+      // Drop data quality indexes
+      LOG.info("Dropping data quality indexes...");
+      dataInsightsApp.deleteDataQualityDataIndex();
+
+      LOG.info("Data Insights indexes and data streams dropped successfully.");
+    } catch (Exception e) {
+      LOG.warn("Failed to drop some Data Insights indexes: {}", e.getMessage());
+      LOG.debug("Data Insights index cleanup error details: ", e);
+    }
+  }
+
   @Command(
       name = "analyze-tables",
       description =