yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-26 09:22:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: Not Include Soft Deleted Assets in DI (#18383)

Browse Source
This commit is contained in:
Mayur Singal 2024-10-24 11:12:50 +05:30 committed by GitHub
parent 51347a981a
commit 913072868a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/DataAssetsWorkflow.java
View File

@ -20,6 +20,7 @@ import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.schema.system.IndexingError;
import org.openmetadata.schema.system.StepStats;
import org.openmetadata.schema.type.Include;
import org.openmetadata.service.apps.bundles.insights.DataInsightsApp;
import org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils;
import org.openmetadata.service.apps.bundles.insights.workflows.WorkflowStats;
@ -28,6 +29,7 @@ import org.openmetadata.service.apps.bundles.insights.workflows.dataAssets.proce
import org.openmetadata.service.apps.bundles.insights.workflows.dataAssets.processors.DataInsightsOpenSearchProcessor;
import org.openmetadata.service.exception.SearchIndexException;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.search.SearchRepository;
import org.openmetadata.service.search.elasticsearch.ElasticSearchIndexSink;
import org.openmetadata.service.search.opensearch.OpenSearchIndexSink;
@ -100,8 +102,15 @@ public class DataAssetsWorkflow {
entityTypes.forEach(
entityType -> {
List<String> fields = List.of("*");
ListFilter filter;
// data product does not support soft delete
if (!entityType.equals("dataProduct")) {
filter = new ListFilter();
} else {
filter = new ListFilter(Include.ALL);
}
PaginatedEntitiesSource source =
new PaginatedEntitiesSource(entityType, batchSize, fields)
new PaginatedEntitiesSource(entityType, batchSize, fields, filter)
.withName(String.format("[DataAssetsWorkflow] %s", entityType));
sources.add(source);
});