yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-29 17:49:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: Not Include Soft Deleted Assets in DI (#18383)

Browse Source
This commit is contained in:
Mayur Singal 2024-10-24 11:12:50 +05:30 committed by ulixius9
parent fca9bb0a4f
commit edc7de7405
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/DataAssetsWorkflow.java
View File

@ -20,6 +20,7 @@ import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.schema.system.IndexingError; import org.openmetadata.schema.system.IndexingError;
import org.openmetadata.schema.system.StepStats; import org.openmetadata.schema.system.StepStats;
import org.openmetadata.schema.type.Include;
import org.openmetadata.service.apps.bundles.insights.DataInsightsApp; import org.openmetadata.service.apps.bundles.insights.DataInsightsApp;
import org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils; import org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils;
import org.openmetadata.service.apps.bundles.insights.workflows.WorkflowStats; import org.openmetadata.service.apps.bundles.insights.workflows.WorkflowStats;
@ -28,6 +29,7 @@ import org.openmetadata.service.apps.bundles.insights.workflows.dataAssets.proce
import org.openmetadata.service.apps.bundles.insights.workflows.dataAssets.processors.DataInsightsOpenSearchProcessor; import org.openmetadata.service.apps.bundles.insights.workflows.dataAssets.processors.DataInsightsOpenSearchProcessor;
import org.openmetadata.service.exception.SearchIndexException; import org.openmetadata.service.exception.SearchIndexException;
import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.SearchRepository;
import org.openmetadata.service.search.elasticsearch.ElasticSearchIndexSink; import org.openmetadata.service.search.elasticsearch.ElasticSearchIndexSink;
import org.openmetadata.service.search.opensearch.OpenSearchIndexSink; import org.openmetadata.service.search.opensearch.OpenSearchIndexSink;
@ -100,8 +102,15 @@ public class DataAssetsWorkflow {
entityTypes.forEach( entityTypes.forEach(
entityType -> { entityType -> {
List<String> fields = List.of("*"); List<String> fields = List.of("*");
ListFilter filter;
// data product does not support soft delete
if (!entityType.equals("dataProduct")) {
filter = new ListFilter();
} else {
filter = new ListFilter(Include.ALL);
}
PaginatedEntitiesSource source = PaginatedEntitiesSource source =
new PaginatedEntitiesSource(entityType, batchSize, fields) new PaginatedEntitiesSource(entityType, batchSize, fields, filter)
.withName(String.format("[DataAssetsWorkflow] %s", entityType)); .withName(String.format("[DataAssetsWorkflow] %s", entityType));
sources.add(source); sources.add(source);
}); });