From a084b6090aff6c5865d65c6e1cec10c9581ba337 Mon Sep 17 00:00:00 2001
From: Pere Miquel Brull <peremiquelbrull@gmail.com>
Date: Tue, 3 Jun 2025 11:28:00 +0200
Subject: [PATCH] FIX CL-1632 - Remove Tier filters from AI agents in AutoPilot
 (#21454)

Co-authored-by: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com>
---
 .../CreateIngestionPipelineImpl.java          |  4 ++--
 .../automatedTask/runApp/RunAppImpl.java      | 21 ++++++++-----------
 2 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateIngestionPipelineImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateIngestionPipelineImpl.java
index a9b63db3da6..eec199a7871 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateIngestionPipelineImpl.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateIngestionPipelineImpl.java
@@ -360,8 +360,8 @@ public class CreateIngestionPipelineImpl {
         .withDatabaseFilterPattern(defaultFilters.get(DATABASE_FILTER_PATTERN))
         .withSchemaFilterPattern(defaultFilters.get(SCHEMA_FILTER_PATTERN))
         .withTableFilterPattern(defaultFilters.get(TABLE_FILTER_PATTERN))
-        .withClassificationFilterPattern(new FilterPattern().withIncludes(DEFAULT_TIERS_TO_PROCESS))
-        .withEnableAutoClassification(true);
+        .withEnableAutoClassification(true)
+        .withStoreSampleData(false);
   }
 
   // Other Services Metadata Pipelines
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java
index 22fc7669b3c..b5213e0d6b1 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java
@@ -103,29 +103,26 @@ public class RunAppImpl {
           .contains(app.getName());
   }
 
+  private String getTableServiceFilter(String serviceName) {
+    return String.format(
+        "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must\":[{\"term\":{\"entityType\":\"table\"}},{\"term\":{\"service.displayName.keyword\":\"%s\"}}]}}]}}}",
+        serviceName);
+  }
+
   private Map<String, Object> getConfig(App app, ServiceEntityInterface service) {
     Object config = JsonUtils.deepCopy(app.getAppConfiguration(), Object.class);
 
     switch (app.getName()) {
       case "CollateAIApplication" -> config =
           (JsonUtils.convertValue(config, CollateAIAppConfig.class))
-              .withFilter(
-                  String.format(
-                      "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"term\":{\"tier.tagFQN\":\"Tier.Tier1\"}},{\"term\":{\"tier.tagFQN\":\"Tier.Tier2\"}}]}},{\"term\":{\"entityType\":\"table\"}},{\"term\":{\"service.displayName.keyword\":\"%s\"}}]}}]}}}",
-                      service.getName()))
+              .withFilter(getTableServiceFilter(service.getName()))
               .withPatchIfEmpty(true);
       case "CollateAIQualityAgentApplication" -> config =
           (JsonUtils.convertValue(config, CollateAIQualityAgentAppConfig.class))
-              .withFilter(
-                  String.format(
-                      "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"term\":{\"tier.tagFQN\":\"Tier.Tier1\"}},{\"term\":{\"tier.tagFQN\":\"Tier.Tier2\"}}]}},{\"term\":{\"entityType\":\"table\"}},{\"term\":{\"service.displayName.keyword\":\"%s\"}}]}}]}}}",
-                      service.getName()));
+              .withFilter(getTableServiceFilter(service.getName()));
       case "CollateAITierAgentApplication" -> config =
           (JsonUtils.convertValue(config, CollateAITierAgentAppConfig.class))
-              .withFilter(
-                  String.format(
-                      "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must\":[{\"term\":{\"entityType\":\"table\"}},{\"term\":{\"service.displayName.keyword\":\"%s\"}}]}}]}}}",
-                      service.getName()))
+              .withFilter(getTableServiceFilter(service.getName()))
               .withPatchIfEmpty(true);
       case "DataInsightsApplication" -> {
         DataInsightsAppConfig updatedAppConfig =