diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateIngestionPipelineImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateIngestionPipelineImpl.java index a9b63db3da6..eec199a7871 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateIngestionPipelineImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipeline/CreateIngestionPipelineImpl.java @@ -360,8 +360,8 @@ public class CreateIngestionPipelineImpl { .withDatabaseFilterPattern(defaultFilters.get(DATABASE_FILTER_PATTERN)) .withSchemaFilterPattern(defaultFilters.get(SCHEMA_FILTER_PATTERN)) .withTableFilterPattern(defaultFilters.get(TABLE_FILTER_PATTERN)) - .withClassificationFilterPattern(new FilterPattern().withIncludes(DEFAULT_TIERS_TO_PROCESS)) - .withEnableAutoClassification(true); + .withEnableAutoClassification(true) + .withStoreSampleData(false); } // Other Services Metadata Pipelines diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java index 22fc7669b3c..b5213e0d6b1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java @@ -103,29 +103,26 @@ public class RunAppImpl { .contains(app.getName()); } + private String getTableServiceFilter(String serviceName) { + return String.format( + "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must\":[{\"term\":{\"entityType\":\"table\"}},{\"term\":{\"service.displayName.keyword\":\"%s\"}}]}}]}}}", + serviceName); + } + private Map getConfig(App app, ServiceEntityInterface service) { Object config = JsonUtils.deepCopy(app.getAppConfiguration(), Object.class); switch (app.getName()) { case "CollateAIApplication" -> config = (JsonUtils.convertValue(config, CollateAIAppConfig.class)) - .withFilter( - String.format( - "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"term\":{\"tier.tagFQN\":\"Tier.Tier1\"}},{\"term\":{\"tier.tagFQN\":\"Tier.Tier2\"}}]}},{\"term\":{\"entityType\":\"table\"}},{\"term\":{\"service.displayName.keyword\":\"%s\"}}]}}]}}}", - service.getName())) + .withFilter(getTableServiceFilter(service.getName())) .withPatchIfEmpty(true); case "CollateAIQualityAgentApplication" -> config = (JsonUtils.convertValue(config, CollateAIQualityAgentAppConfig.class)) - .withFilter( - String.format( - "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"term\":{\"tier.tagFQN\":\"Tier.Tier1\"}},{\"term\":{\"tier.tagFQN\":\"Tier.Tier2\"}}]}},{\"term\":{\"entityType\":\"table\"}},{\"term\":{\"service.displayName.keyword\":\"%s\"}}]}}]}}}", - service.getName())); + .withFilter(getTableServiceFilter(service.getName())); case "CollateAITierAgentApplication" -> config = (JsonUtils.convertValue(config, CollateAITierAgentAppConfig.class)) - .withFilter( - String.format( - "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must\":[{\"term\":{\"entityType\":\"table\"}},{\"term\":{\"service.displayName.keyword\":\"%s\"}}]}}]}}}", - service.getName())) + .withFilter(getTableServiceFilter(service.getName())) .withPatchIfEmpty(true); case "DataInsightsApplication" -> { DataInsightsAppConfig updatedAppConfig =