From 3cf928bcc706308e8cedcf48e097532c6f11b3c2 Mon Sep 17 00:00:00 2001 From: Teddy Date: Wed, 15 May 2024 10:38:58 +0200 Subject: [PATCH] MINOR - Test Case Incident Manager Reindex (#16123) * feat: added reindex logic for test cases * fix: broaden exception to catch ES/OS failure * style: ran java linting * fix: update buildESDoc method * style: ran java linting * fix ui tests * fix: update HashMap to Map --------- Co-authored-by: Chira Madlani --- .../bundles/searchIndex/SearchIndexApp.java | 198 ++++++++++-------- .../service/jdbi3/EntityTimeSeriesDAO.java | 16 ++ .../jdbi3/EntityTimeSeriesRepository.java | 124 ++++++++--- .../service/jdbi3/TestSuiteRepository.java | 3 +- ...asticSearchEntityTimeSeriesProcessor.java} | 28 +-- ... OpenSearchEntityTimeSeriesProcessor.java} | 28 +-- .../service/workflows/interfaces/Source.java | 8 + .../PaginatedDataInsightSource.java | 150 ------------- .../searchIndex/PaginatedEntitiesSource.java | 7 +- .../PaginatedEntityTimeSeriesSource.java | 141 +++++++++++++ .../workflows/searchIndex/ReindexingUtil.java | 17 +- .../data/app/SearchIndexingApplication.json | 3 +- .../ui/cypress/common/Entities/EntityClass.ts | 2 +- .../resources/ui/src/enums/entity.enum.ts | 1 + .../ui/src/interface/search.interface.ts | 11 + .../SearchIndexingApplication.json | 6 +- .../src/utils/mocks/ApplicationUtils.mock.ts | 11 + 17 files changed, 454 insertions(+), 300 deletions(-) rename openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/{ElasticSearchDataInsightProcessor.java => ElasticSearchEntityTimeSeriesProcessor.java} (78%) rename openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/{OpenSearchDataInsightProcessor.java => OpenSearchEntityTimeSeriesProcessor.java} (78%) delete mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedDataInsightSource.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntityTimeSeriesSource.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index 20f1ec24336..04dfdda0a0d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -6,20 +6,18 @@ import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_NAME_LIST_KEY; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_TYPE_KEY; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getTotalRequestToProcess; -import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.isDataInsightIndex; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.exception.ExceptionUtils; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.EntityInterface; -import org.openmetadata.schema.analytics.ReportData; +import org.openmetadata.schema.EntityTimeSeriesInterface; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppRunRecord; import org.openmetadata.schema.entity.app.FailureContext; @@ -33,20 +31,21 @@ import org.openmetadata.service.apps.AbstractNativeApplication; import org.openmetadata.service.exception.SearchIndexException; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.search.SearchRepository; -import org.openmetadata.service.search.elasticsearch.ElasticSearchDataInsightProcessor; import org.openmetadata.service.search.elasticsearch.ElasticSearchEntitiesProcessor; +import org.openmetadata.service.search.elasticsearch.ElasticSearchEntityTimeSeriesProcessor; import org.openmetadata.service.search.elasticsearch.ElasticSearchIndexSink; import org.openmetadata.service.search.models.IndexMapping; -import org.openmetadata.service.search.opensearch.OpenSearchDataInsightProcessor; import org.openmetadata.service.search.opensearch.OpenSearchEntitiesProcessor; +import org.openmetadata.service.search.opensearch.OpenSearchEntityTimeSeriesProcessor; import org.openmetadata.service.search.opensearch.OpenSearchIndexSink; import org.openmetadata.service.socket.WebSocketManager; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.ResultList; import org.openmetadata.service.workflows.interfaces.Processor; import org.openmetadata.service.workflows.interfaces.Sink; -import org.openmetadata.service.workflows.searchIndex.PaginatedDataInsightSource; +import org.openmetadata.service.workflows.interfaces.Source; import org.openmetadata.service.workflows.searchIndex.PaginatedEntitiesSource; +import org.openmetadata.service.workflows.searchIndex.PaginatedEntityTimeSeriesSource; import org.quartz.JobExecutionContext; @Slf4j @@ -88,11 +87,17 @@ public class SearchIndexApp extends AbstractNativeApplication { "webAnalyticUserActivityReportData", "domain", "storedProcedure", - "storageService"); - private final List paginatedEntitiesSources = new ArrayList<>(); - private final List paginatedDataInsightSources = new ArrayList<>(); + "storageService", + "testCaseResolutionStatus"); + public static final Set TIME_SERIES_ENTITIES = + Set.of( + "entityReportData", + "webAnalyticEntityViewReportData", + "webAnalyticUserActivityReportData", + "testCaseResolutionStatus"); + private final List paginatedSources = new ArrayList<>(); private Processor entityProcessor; - private Processor dataInsightProcessor; + private Processor entityTimeSeriesProcessor; private Sink searchIndexSink; @Getter EventPublisherJob jobData; @@ -133,8 +138,7 @@ public class SearchIndexApp extends AbstractNativeApplication { } // Run ReIndexing - entitiesReIndex(jobExecutionContext); - dataInsightReindex(jobExecutionContext); + performReindex(jobExecutionContext); // Mark Job as Completed updateJobStatus(); } catch (Exception ex) { @@ -167,27 +171,31 @@ public class SearchIndexApp extends AbstractNativeApplication { .getEntities() .forEach( entityType -> { - if (!isDataInsightIndex(entityType)) { + if (!TIME_SERIES_ENTITIES.contains(entityType)) { List fields = List.of("*"); PaginatedEntitiesSource source = new PaginatedEntitiesSource(entityType, jobData.getBatchSize(), fields); if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) { source.setCursor(jobData.getAfterCursor()); } - paginatedEntitiesSources.add(source); + paginatedSources.add(source); } else { - paginatedDataInsightSources.add( - new PaginatedDataInsightSource( - collectionDAO, entityType, jobData.getBatchSize())); + PaginatedEntityTimeSeriesSource source = + new PaginatedEntityTimeSeriesSource( + entityType, jobData.getBatchSize(), List.of("*")); + if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) { + source.setCursor(jobData.getAfterCursor()); + } + paginatedSources.add(source); } }); if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) { this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords); - this.dataInsightProcessor = new OpenSearchDataInsightProcessor(totalRecords); + this.entityTimeSeriesProcessor = new OpenSearchEntityTimeSeriesProcessor(totalRecords); this.searchIndexSink = new OpenSearchIndexSink(searchRepository, totalRecords); } else { this.entityProcessor = new ElasticSearchEntitiesProcessor(totalRecords); - this.dataInsightProcessor = new ElasticSearchDataInsightProcessor(totalRecords); + this.entityTimeSeriesProcessor = new ElasticSearchEntityTimeSeriesProcessor(totalRecords); this.searchIndexSink = new ElasticSearchIndexSink(searchRepository, totalRecords); } } @@ -213,89 +221,107 @@ public class SearchIndexApp extends AbstractNativeApplication { pushAppStatusUpdates(jobExecutionContext, appRecord, true); } - private void entitiesReIndex(JobExecutionContext jobExecutionContext) { + private void performReindex(JobExecutionContext jobExecutionContext) throws SearchIndexException { Map contextData = new HashMap<>(); - for (PaginatedEntitiesSource paginatedEntitiesSource : paginatedEntitiesSources) { - reCreateIndexes(paginatedEntitiesSource.getEntityType()); - contextData.put(ENTITY_TYPE_KEY, paginatedEntitiesSource.getEntityType()); - ResultList resultList; - while (!stopped && !paginatedEntitiesSource.isDone()) { + for (Source paginatedSource : paginatedSources) { + List entityName = new ArrayList<>(); + reCreateIndexes(paginatedSource.getEntityType()); + contextData.put(ENTITY_TYPE_KEY, paginatedSource.getEntityType()); + Object resultList; + while (!stopped && !paginatedSource.isDone()) { try { - resultList = paginatedEntitiesSource.readNext(null); - List entityName = - resultList.getData().stream() - .map( - entity -> - String.format( - "%s %s", - paginatedEntitiesSource.getEntityType(), - entity.getFullyQualifiedName())) - .collect(Collectors.toList()); - - contextData.put(ENTITY_NAME_LIST_KEY, entityName); - if (!resultList.getData().isEmpty()) { - searchIndexSink.write(entityProcessor.process(resultList, contextData), contextData); - if (!resultList.getErrors().isEmpty()) { - throw new SearchIndexException( - new IndexingError() - .withErrorSource(READER) - .withLastFailedCursor(paginatedEntitiesSource.getLastFailedCursor()) - .withSubmittedCount(paginatedEntitiesSource.getBatchSize()) - .withSuccessCount(resultList.getData().size()) - .withFailedCount(resultList.getErrors().size()) - .withMessage( - "Issues in Reading A Batch For Entities. Check Errors Corresponding to Entities.") - .withFailedEntities(resultList.getErrors())); - } - paginatedEntitiesSource.updateStats(resultList.getData().size(), 0); + resultList = paginatedSource.readNext(null); + if (!TIME_SERIES_ENTITIES.contains(paginatedSource.getEntityType())) { + entityName = + getEntityNameFromEntity( + (ResultList) resultList, + paginatedSource.getEntityType()); + contextData.put(ENTITY_NAME_LIST_KEY, entityName); + processEntity( + (ResultList) resultList, contextData, paginatedSource); + } else { + entityName = + getEntityNameFromEntityTimeSeries( + (ResultList) resultList, + paginatedSource.getEntityType()); + contextData.put(ENTITY_NAME_LIST_KEY, entityName); + processEntityTimeSeries( + (ResultList) resultList, + contextData, + paginatedSource); } + } catch (SearchIndexException rx) { jobData.setStatus(EventPublisherJob.Status.FAILED); jobData.setFailure(rx.getIndexingError()); - paginatedEntitiesSource.updateStats( + paginatedSource.updateStats( rx.getIndexingError().getSuccessCount(), rx.getIndexingError().getFailedCount()); } finally { - updateStats(paginatedEntitiesSource.getEntityType(), paginatedEntitiesSource.getStats()); + updateStats(paginatedSource.getEntityType(), paginatedSource.getStats()); sendUpdates(jobExecutionContext); } } } } - private void dataInsightReindex(JobExecutionContext jobExecutionContext) { - Map contextData = new HashMap<>(); - for (PaginatedDataInsightSource paginatedDataInsightSource : paginatedDataInsightSources) { - reCreateIndexes(paginatedDataInsightSource.getEntityType()); - contextData.put(ENTITY_TYPE_KEY, paginatedDataInsightSource.getEntityType()); - ResultList resultList; - while (!stopped && !paginatedDataInsightSource.isDone()) { - try { - resultList = paginatedDataInsightSource.readNext(null); - List entityName = - resultList.getData().stream() - .map( - entity -> - String.format( - "%s %s", paginatedDataInsightSource.getEntityType(), entity.getId())) - .collect(Collectors.toList()); + private List getEntityNameFromEntity( + ResultList resultList, String entityType) { + return resultList.getData().stream() + .map(entity -> String.format("%s %s", entityType, entity.getId())) + .toList(); + } - contextData.put(ENTITY_NAME_LIST_KEY, entityName); - if (!resultList.getData().isEmpty()) { - searchIndexSink.write( - dataInsightProcessor.process(resultList, contextData), contextData); - } - paginatedDataInsightSource.updateStats(resultList.getData().size(), 0); - } catch (SearchIndexException ex) { - jobData.setStatus(EventPublisherJob.Status.FAILED); - jobData.setFailure(ex.getIndexingError()); - paginatedDataInsightSource.updateStats( - ex.getIndexingError().getSuccessCount(), ex.getIndexingError().getFailedCount()); - } finally { - updateStats( - paginatedDataInsightSource.getEntityType(), paginatedDataInsightSource.getStats()); - sendUpdates(jobExecutionContext); - } + private List getEntityNameFromEntityTimeSeries( + ResultList resultList, String entityType) { + return resultList.getData().stream() + .map(entity -> String.format("%s %s", entityType, entity.getId())) + .toList(); + } + + private void processEntity( + ResultList resultList, + Map contextData, + Source paginatedSource) + throws SearchIndexException { + if (!resultList.getData().isEmpty()) { + searchIndexSink.write(entityProcessor.process(resultList, contextData), contextData); + if (!resultList.getErrors().isEmpty()) { + throw new SearchIndexException( + new IndexingError() + .withErrorSource(READER) + .withLastFailedCursor(paginatedSource.getLastFailedCursor()) + .withSubmittedCount(paginatedSource.getBatchSize()) + .withSuccessCount(resultList.getData().size()) + .withFailedCount(resultList.getErrors().size()) + .withMessage( + "Issues in Reading A Batch For Entities. Check Errors Corresponding to Entities.") + .withFailedEntities(resultList.getErrors())); } + paginatedSource.updateStats(resultList.getData().size(), 0); + } + } + + private void processEntityTimeSeries( + ResultList resultList, + Map contextData, + Source paginatedSource) + throws SearchIndexException { + if (!resultList.getData().isEmpty()) { + searchIndexSink.write( + entityTimeSeriesProcessor.process(resultList, contextData), contextData); + if (!resultList.getErrors().isEmpty()) { + throw new SearchIndexException( + new IndexingError() + .withErrorSource(READER) + .withLastFailedCursor(paginatedSource.getLastFailedCursor()) + .withSubmittedCount(paginatedSource.getBatchSize()) + .withSuccessCount(resultList.getData().size()) + .withFailedCount(resultList.getErrors().size()) + .withMessage( + "Issues in Reading A Batch For Entities. Check Errors Corresponding to Entities.") + .withFailedEntities(resultList.getErrors())); + } + paginatedSource.updateStats(resultList.getData().size(), 0); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesDAO.java index f6f2520a3f4..73f0abb668b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesDAO.java @@ -115,6 +115,14 @@ public interface EntityTimeSeriesDAO { update(getTimeSeriesTableName(), json, id.toString()); } + @SqlQuery( + "SELECT json FROM " + "ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") + List listWithOffset( + @Define("table") String table, + @Define("cond") String cond, + @Bind("limit") int limit, + @Bind("offset") int offset); + @SqlQuery( "SELECT json FROM
" + "AND timestamp BETWEEN :startTs AND :endTs " @@ -158,6 +166,10 @@ public interface EntityTimeSeriesDAO { getTimeSeriesTableName(), filter.getCondition(), limit, offset, startTs, endTs); } + default List listWithOffset(ListFilter filter, int limit, int offset) { + return listWithOffset(getTimeSeriesTableName(), filter.getCondition(), limit, offset); + } + @ConnectionAwareSqlUpdate( value = "UPDATE
set json = :json where entityFQNHash=:entityFQNHash and extension=:extension and timestamp=:timestamp and json -> '$.operation' = :operation", @@ -198,6 +210,10 @@ public interface EntityTimeSeriesDAO { return listCount(getTimeSeriesTableName(), filter.getCondition()); } + default int listCount() { + return listCount(new ListFilter(null)); + } + @SqlQuery("SELECT count(*) FROM
AND timestamp BETWEEN :startTs AND :endTs") int listCount( @Define("table") String table, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java index fe8375967f3..5766e97a5ee 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java @@ -3,11 +3,14 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.schema.type.Include.ALL; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import lombok.Getter; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.schema.EntityTimeSeriesInterface; +import org.openmetadata.schema.system.EntityError; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; @@ -129,9 +132,21 @@ public abstract class EntityTimeSeriesRepository getResultList( + List entities, + String beforeCursor, + String afterCursor, + int total, + List errors) { + if (errors == null) { + return new ResultList<>(entities, beforeCursor, afterCursor, total); + } + return new ResultList<>(entities, errors, beforeCursor, afterCursor, total); + } + public final ResultList getResultList( List entities, String beforeCursor, String afterCursor, int total) { - return new ResultList<>(entities, beforeCursor, afterCursor, total); + return getResultList(entities, beforeCursor, afterCursor, total, null); } /** @@ -139,36 +154,52 @@ public abstract class EntityTimeSeriesRepository listWithOffset( - String offset, ListFilter filter, int limitParam, Long startTs, Long endTs, boolean latest) { + public ResultList listWithOffset( + String offset, + ListFilter filter, + int limitParam, + Long startTs, + Long endTs, + boolean latest, + boolean skipErrors) { int total = timeSeriesDao.listCount(filter, startTs, endTs, latest); List entityList = new ArrayList<>(); - - int offsetInt = offset != null ? Integer.parseInt(RestUtil.decodeCursor(offset)) : 0; - int afterOffsetInt = offsetInt + limitParam; - int beforeOffsetInt = offsetInt - limitParam; - - // If offset is negative, then set it to 0 if you pass offset 4 and limit 10, then the previous - // page will be at offset 0 - if (beforeOffsetInt < 0) beforeOffsetInt = 0; - - // if offsetInt is 0 (i.e. either no offset or offset is 0), then set it to null as there is no - // previous page - String beforeOffset = (offsetInt == 0) ? null : String.valueOf(beforeOffsetInt); - - // If afterOffset is greater than total, then set it to null to indicate end of list - String afterOffset = afterOffsetInt >= total ? null : String.valueOf(afterOffsetInt); + List errors = null; + int offsetInt = getOffset(offset); + String afterOffset = getAfterOffset(offsetInt, limitParam, total); + String beforeOffset = getBeforeOffset(offsetInt, limitParam); if (limitParam > 0) { List jsons = timeSeriesDao.listWithOffset(filter, limitParam, offsetInt, startTs, endTs, latest); - - for (String json : jsons) { - T recordEntity = JsonUtils.readValue(json, entityClass); - setInheritedFields(recordEntity); - entityList.add(recordEntity); + Map> entityListMap = getEntityList(jsons, skipErrors); + entityList = (List) entityListMap.get("entityList"); + if (skipErrors) { + errors = (List) entityListMap.get("errors"); } - return getResultList(entityList, beforeOffset, afterOffset, total); + return getResultList(entityList, beforeOffset, afterOffset, total, errors); + } else { + return getResultList(entityList, null, null, total); + } + } + + public ResultList listWithOffset( + String offset, ListFilter filter, int limitParam, boolean skipErrors) { + int total = timeSeriesDao.listCount(filter); + List entityList = new ArrayList<>(); + List errors = null; + + int offsetInt = getOffset(offset); + String afterOffset = getAfterOffset(offsetInt, limitParam, total); + String beforeOffset = getBeforeOffset(offsetInt, limitParam); + if (limitParam > 0) { + List jsons = timeSeriesDao.listWithOffset(filter, limitParam, offsetInt); + Map> entityListMap = getEntityList(jsons, skipErrors); + entityList = (List) entityListMap.get("entityList"); + if (skipErrors) { + errors = (List) entityListMap.get("errors"); + } + return getResultList(entityList, beforeOffset, afterOffset, total, errors); } else { return getResultList(entityList, null, null, total); } @@ -176,7 +207,7 @@ public abstract class EntityTimeSeriesRepository list( String offset, Long startTs, Long endTs, int limitParam, ListFilter filter, boolean latest) { - return listWithOffset(offset, filter, limitParam, startTs, endTs, latest); + return listWithOffset(offset, filter, limitParam, startTs, endTs, latest, false); } public T getLatestRecord(String recordFQN) { @@ -212,4 +243,47 @@ public abstract class EntityTimeSeriesRepository= total ? null : String.valueOf(afterOffset); + } + + private String getBeforeOffset(int offsetInt, int limit) { + int beforeOffsetInt = offsetInt - limit; + // If offset is negative, then set it to 0 if you pass offset 4 and limit 10, then the previous + // page will be at offset 0 + if (beforeOffsetInt < 0) beforeOffsetInt = 0; + // if offsetInt is 0 (i.e. either no offset or offset is 0), then set it to null as there is no + // previous page + return (offsetInt == 0) ? null : String.valueOf(beforeOffsetInt); + } + + private int getOffset(String offset) { + return offset != null ? Integer.parseInt(RestUtil.decodeCursor(offset)) : 0; + } + + private Map> getEntityList(List jsons, boolean skipErrors) { + List entityList = new ArrayList<>(); + List errors = new ArrayList<>(); + + Map> resultList = new HashMap<>(); + + for (String json : jsons) { + try { + T recordEntity = JsonUtils.readValue(json, entityClass); + setInheritedFields(recordEntity); + entityList.add(recordEntity); + } catch (Exception e) { + if (!skipErrors) { + throw e; + } + errors.add(new EntityError().withMessage(e.getMessage())); + } + } + resultList.put("entityList", entityList); + resultList.put("errors", errors); + return resultList; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java index ffa249a009b..6cd7e36e834 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java @@ -9,7 +9,6 @@ import static org.openmetadata.service.Entity.TEST_CASE; import static org.openmetadata.service.Entity.TEST_SUITE; import static org.openmetadata.service.util.FullyQualifiedName.quoteName; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -232,7 +231,7 @@ public class TestSuiteRepository extends EntityRepository { testSummary = getEntityTestCasesExecutionSummary(testCaseResultSummary); } return testSummary; - } catch (IOException e) { + } catch (Exception e) { LOG.error("Error reading aggregation query", e); } return null; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntityTimeSeriesProcessor.java similarity index 78% rename from openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java rename to openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntityTimeSeriesProcessor.java index d10afe6a18b..f5902350232 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntityTimeSeriesProcessor.java @@ -8,31 +8,32 @@ import es.org.elasticsearch.action.update.UpdateRequest; import es.org.elasticsearch.xcontent.XContentType; import java.util.List; import java.util.Map; +import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.glassfish.jersey.internal.util.ExceptionUtils; import org.openmetadata.common.utils.CommonUtil; -import org.openmetadata.schema.analytics.ReportData; +import org.openmetadata.schema.EntityTimeSeriesInterface; import org.openmetadata.schema.system.IndexingError; import org.openmetadata.schema.system.StepStats; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.SearchIndexException; -import org.openmetadata.service.search.indexes.ReportDataIndexes; import org.openmetadata.service.search.models.IndexMapping; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.ResultList; import org.openmetadata.service.workflows.interfaces.Processor; @Slf4j -public class ElasticSearchDataInsightProcessor - implements Processor> { +public class ElasticSearchEntityTimeSeriesProcessor + implements Processor> { private final StepStats stats = new StepStats(); - public ElasticSearchDataInsightProcessor(int total) { + public ElasticSearchEntityTimeSeriesProcessor(int total) { this.stats.withTotalRecords(total).withSuccessRecords(0).withFailedRecords(0); } @Override - public BulkRequest process(ResultList input, Map contextData) + public BulkRequest process( + ResultList input, Map contextData) throws SearchIndexException { String entityType = (String) contextData.get(ENTITY_TYPE_KEY); if (CommonUtil.nullOrEmpty(entityType)) { @@ -70,23 +71,26 @@ public class ElasticSearchDataInsightProcessor return requests; } - private BulkRequest buildBulkRequests(String entityType, List entities) { + private BulkRequest buildBulkRequests( + String entityType, List entities) { BulkRequest bulkRequests = new BulkRequest(); - for (ReportData reportData : entities) { - UpdateRequest request = getUpdateRequest(entityType, reportData); + for (EntityTimeSeriesInterface entity : entities) { + UpdateRequest request = getUpdateRequest(entityType, entity); bulkRequests.add(request); } return bulkRequests; } - private UpdateRequest getUpdateRequest(String entityType, ReportData reportData) { + private UpdateRequest getUpdateRequest(String entityType, EntityTimeSeriesInterface entity) { IndexMapping indexMapping = Entity.getSearchRepository().getIndexMapping(entityType); UpdateRequest updateRequest = new UpdateRequest( indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()), - reportData.getId().toString()); + entity.getId().toString()); updateRequest.doc( - JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildSearchIndexDoc()), + JsonUtils.pojoToJson( + Objects.requireNonNull( + Entity.buildSearchIndex(entityType, entity).buildSearchIndexDoc())), XContentType.JSON); updateRequest.docAsUpsert(true); return updateRequest; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntityTimeSeriesProcessor.java similarity index 78% rename from openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java rename to openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntityTimeSeriesProcessor.java index d6dc8195832..8a6a79f12f0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntityTimeSeriesProcessor.java @@ -5,15 +5,15 @@ import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getU import java.util.List; import java.util.Map; +import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.glassfish.jersey.internal.util.ExceptionUtils; import org.openmetadata.common.utils.CommonUtil; -import org.openmetadata.schema.analytics.ReportData; +import org.openmetadata.schema.EntityTimeSeriesInterface; import org.openmetadata.schema.system.IndexingError; import org.openmetadata.schema.system.StepStats; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.SearchIndexException; -import org.openmetadata.service.search.indexes.ReportDataIndexes; import org.openmetadata.service.search.models.IndexMapping; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.ResultList; @@ -23,16 +23,17 @@ import os.org.opensearch.action.update.UpdateRequest; import os.org.opensearch.common.xcontent.XContentType; @Slf4j -public class OpenSearchDataInsightProcessor - implements Processor> { +public class OpenSearchEntityTimeSeriesProcessor + implements Processor> { private final StepStats stats = new StepStats(); - public OpenSearchDataInsightProcessor(int total) { + public OpenSearchEntityTimeSeriesProcessor(int total) { this.stats.withTotalRecords(total).withSuccessRecords(0).withFailedRecords(0); } @Override - public BulkRequest process(ResultList input, Map contextData) + public BulkRequest process( + ResultList input, Map contextData) throws SearchIndexException { String entityType = (String) contextData.get(ENTITY_TYPE_KEY); if (CommonUtil.nullOrEmpty(entityType)) { @@ -70,23 +71,26 @@ public class OpenSearchDataInsightProcessor return requests; } - private BulkRequest buildBulkRequests(String entityType, List entities) { + private BulkRequest buildBulkRequests( + String entityType, List entities) { BulkRequest bulkRequests = new BulkRequest(); - for (ReportData reportData : entities) { - UpdateRequest request = getUpdateRequest(entityType, reportData); + for (EntityTimeSeriesInterface entity : entities) { + UpdateRequest request = getUpdateRequest(entityType, entity); bulkRequests.add(request); } return bulkRequests; } - private UpdateRequest getUpdateRequest(String entityType, ReportData reportData) { + private UpdateRequest getUpdateRequest(String entityType, EntityTimeSeriesInterface entity) { IndexMapping indexMapping = Entity.getSearchRepository().getIndexMapping(entityType); UpdateRequest updateRequest = new UpdateRequest( indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()), - reportData.getId().toString()); + entity.getId().toString()); updateRequest.doc( - JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildSearchIndexDoc()), + JsonUtils.pojoToJson( + Objects.requireNonNull( + Entity.buildSearchIndex(entityType, entity).buildSearchIndexDoc())), XContentType.JSON); updateRequest.docAsUpsert(true); return updateRequest; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/interfaces/Source.java b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/interfaces/Source.java index 145fec91193..ca59c04d14f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/interfaces/Source.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/interfaces/Source.java @@ -23,4 +23,12 @@ public interface Source extends Stats { List getReaderErrors(); void reset(); + + String getEntityType(); + + int getBatchSize(); + + String getLastFailedCursor(); + + boolean isDone(); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedDataInsightSource.java b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedDataInsightSource.java deleted file mode 100644 index b4e7c9f46c4..00000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedDataInsightSource.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2022 Collate - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.openmetadata.service.workflows.searchIndex; - -import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.glassfish.jersey.internal.util.ExceptionUtils; -import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.schema.system.IndexingError; -import org.openmetadata.schema.system.StepStats; -import org.openmetadata.service.exception.SearchIndexException; -import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.ListFilter; -import org.openmetadata.service.util.JsonUtils; -import org.openmetadata.service.util.RestUtil; -import org.openmetadata.service.util.ResultList; -import org.openmetadata.service.workflows.interfaces.Source; - -@Slf4j -public class PaginatedDataInsightSource implements Source> { - private final CollectionDAO dao; - @Getter private final String entityType; - @Getter private final int batchSize; - @Getter private final List readerErrors = new ArrayList<>(); - @Getter private final StepStats stats = new StepStats(); - private String cursor = null; - @Getter private boolean isDone = false; - - public PaginatedDataInsightSource(CollectionDAO dao, String entityType, int batchSize) { - this.dao = dao; - this.entityType = entityType; - this.batchSize = batchSize; - this.stats - .withTotalRecords( - dao.reportDataTimeSeriesDao() - .listCount(new ListFilter(null).addQueryParam("entityFQNHash", entityType))) - .withSuccessRecords(0) - .withFailedRecords(0); - } - - @Override - public ResultList readNext(Map contextData) - throws SearchIndexException { - if (!isDone) { - ResultList data = read(cursor); - cursor = data.getPaging().getAfter(); - if (cursor == null) { - isDone = true; - } - return data; - } - return null; - } - - @Override - public void reset() { - cursor = null; - isDone = false; - } - - private ResultList read(String afterCursor) throws SearchIndexException { - LOG.debug("[DataInsightReader] Fetching a Batch of Size: {} ", batchSize); - ResultList result = null; - try { - result = getReportDataPagination(entityType, batchSize, afterCursor); - LOG.debug( - "[DataInsightReader] Batch Stats :- Submitted : {} Success: {} Failed: {}", - batchSize, - result.getData().size(), - 0); - // updateStats(result.getData().size(), result.getErrors().size()); - } catch (Exception ex) { - IndexingError indexingError = - new IndexingError() - .withErrorSource(IndexingError.ErrorSource.READER) - .withSubmittedCount(batchSize) - .withSuccessCount(0) - .withFailedCount(batchSize) - .withMessage("Issues in Reading A Batch For Data Insight Data.") - .withStackTrace(ExceptionUtils.exceptionStackTraceAsString(ex)); - LOG.debug( - "[DataInsightReader] Failing Completely. Details : {}", - JsonUtils.pojoToJson(indexingError)); - if (result != null) { - if (result.getPaging().getAfter() == null) { - isDone = true; - int recordToRead = - stats.getTotalRecords() - (stats.getSuccessRecords() + stats.getFailedRecords()); - updateStats(result.getData().size(), recordToRead - result.getData().size()); - } else { - updateStats(result.getData().size(), batchSize - result.getData().size()); - } - } else { - updateStats(0, batchSize); - } - - throw new SearchIndexException(indexingError); - } - - return result; - } - - public ResultList getReportDataPagination(String entityFQN, int limit, String after) { - int reportDataCount = - dao.reportDataTimeSeriesDao() - .listCount(new ListFilter(null).addQueryParam("entityFQNHash", entityFQN)); - List reportDataList = - dao.reportDataTimeSeriesDao() - .getAfterExtension( - entityFQN, limit + 1, after == null ? "0" : RestUtil.decodeCursor(after)); - return getAfterExtensionList(reportDataList, after, limit, reportDataCount); - } - - private ResultList getAfterExtensionList( - List reportDataRowList, String after, int limit, int total) { - String beforeCursor; - String afterCursor = null; - beforeCursor = after == null ? null : reportDataRowList.get(0).getRowNum(); - if (reportDataRowList.size() > limit) { - reportDataRowList.remove(limit); - afterCursor = reportDataRowList.get(limit - 1).getRowNum(); - } - List reportDataList = new ArrayList<>(); - for (CollectionDAO.ReportDataRow reportDataRow : reportDataRowList) { - reportDataList.add(reportDataRow.getReportData()); - } - return new ResultList<>(reportDataList, new ArrayList<>(), beforeCursor, afterCursor, total); - } - - @Override - public void updateStats(int currentSuccess, int currentFailed) { - getUpdatedStats(stats, currentSuccess, currentFailed); - } -} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java index c3c17a153d4..642b7c4a349 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.glassfish.jersey.internal.util.ExceptionUtils; import org.openmetadata.schema.EntityInterface; @@ -42,7 +43,7 @@ public class PaginatedEntitiesSource implements Source readerErrors = new ArrayList<>(); @Getter private final StepStats stats = new StepStats(); @Getter private String lastFailedCursor = null; - private String cursor = RestUtil.encodeCursor("0"); + @Setter private String cursor = RestUtil.encodeCursor("0"); @Getter private boolean isDone = false; public PaginatedEntitiesSource(String entityType, int batchSize, List fields) { @@ -139,8 +140,4 @@ public class PaginatedEntitiesSource implements Source> { + @Getter private final int batchSize; + @Getter private final String entityType; + @Getter private final List fields; + @Getter private final List readerErrors = new ArrayList<>(); + @Getter private final StepStats stats = new StepStats(); + @Getter private String lastFailedCursor = null; + @Setter private String cursor = RestUtil.encodeCursor("0"); + @Getter private boolean isDone = false; + + public PaginatedEntityTimeSeriesSource(String entityType, int batchSize, List fields) { + this.entityType = entityType; + this.batchSize = batchSize; + this.fields = fields; + this.stats + .withTotalRecords(getEntityTimeSeriesRepository().getTimeSeriesDao().listCount(getFilter())) + .withSuccessRecords(0) + .withFailedRecords(0); + } + + @Override + public ResultList readNext(Map contextData) + throws SearchIndexException { + ResultList data = null; + if (!isDone) { + data = read(cursor); + cursor = data.getPaging().getAfter(); + if (cursor == null) { + isDone = true; + } + } + return data; + } + + private ResultList read(String cursor) + throws SearchIndexException { + LOG.debug("[PaginatedEntityTimeSeriesSource] Fetching a Batch of Size: {} ", batchSize); + EntityTimeSeriesRepository repository = + getEntityTimeSeriesRepository(); + ResultList result; + ListFilter filter = getFilter(); + try { + result = repository.listWithOffset(cursor, filter, batchSize, true); + if (!result.getErrors().isEmpty()) { + lastFailedCursor = this.cursor; + if (result.getPaging().getAfter() == null) { + isDone = true; + } else { + this.cursor = result.getPaging().getAfter(); + } + return result; + } + LOG.debug( + "[PaginatedEntitiesSource] Batch Stats :- %n Submitted : {} Success: {} Failed: {}", + batchSize, result.getData().size(), result.getErrors().size()); + } catch (Exception e) { + lastFailedCursor = this.cursor; + int remainingRecords = + stats.getTotalRecords() - stats.getFailedRecords() - stats.getSuccessRecords(); + int submittedRecords; + if (remainingRecords - batchSize <= 0) { + submittedRecords = remainingRecords; + updateStats(0, remainingRecords); + this.cursor = null; + this.isDone = true; + } else { + submittedRecords = batchSize; + String decodedCursor = RestUtil.decodeCursor(cursor); + this.cursor = + RestUtil.encodeCursor(String.valueOf(Integer.parseInt(decodedCursor) + batchSize)); + updateStats(0, batchSize); + } + IndexingError indexingError = + new IndexingError() + .withErrorSource(READER) + .withSubmittedCount(submittedRecords) + .withSuccessCount(0) + .withFailedCount(submittedRecords) + .withMessage( + "Issues in Reading A Batch For Entities. No Relationship Issue , Json Processing or DB issue.") + .withLastFailedCursor(lastFailedCursor) + .withStackTrace(ExceptionUtils.exceptionStackTraceAsString(e)); + LOG.debug(indexingError.getMessage()); + throw new SearchIndexException(indexingError); + } + return result; + } + + @Override + public void reset() { + cursor = null; + isDone = false; + } + + @Override + public void updateStats(int currentSuccess, int currentFailed) { + getUpdatedStats(stats, currentSuccess, currentFailed); + } + + private ListFilter getFilter() { + ListFilter filter = new ListFilter(null); + if (ReindexingUtil.isDataInsightIndex(entityType)) { + filter.addQueryParam("entityFQNHash", entityType); + } + return filter; + } + + private EntityTimeSeriesRepository + getEntityTimeSeriesRepository() { + if (ReindexingUtil.isDataInsightIndex(entityType)) { + return Entity.getEntityTimeSeriesRepository(Entity.ENTITY_REPORT_DATA); + } else { + return Entity.getEntityTimeSeriesRepository(entityType); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/ReindexingUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/ReindexingUtil.java index c03ace2401a..06f8b697cf5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/ReindexingUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/ReindexingUtil.java @@ -13,6 +13,8 @@ package org.openmetadata.service.workflows.searchIndex; +import static org.openmetadata.service.apps.bundles.searchIndex.SearchIndexApp.TIME_SERIES_ENTITIES; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import java.util.ArrayList; @@ -29,6 +31,7 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.EntityTimeSeriesRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.util.JsonUtils; @@ -55,13 +58,19 @@ public class ReindexingUtil { public static int getTotalRequestToProcess(Set entities, CollectionDAO dao) { int total = 0; for (String entityType : entities) { - if (!isDataInsightIndex(entityType)) { + if (!TIME_SERIES_ENTITIES.contains(entityType)) { EntityRepository repository = Entity.getEntityRepository(entityType); total += repository.getDao().listTotalCount(); } else { - total += - dao.reportDataTimeSeriesDao() - .listCount(new ListFilter(null).addQueryParam("entityFQNHash", entityType)); + EntityTimeSeriesRepository repository; + ListFilter listFilter = new ListFilter(null); + if (isDataInsightIndex(entityType)) { + listFilter.addQueryParam("entityFQNHash", entityType); + repository = Entity.getEntityTimeSeriesRepository(Entity.ENTITY_REPORT_DATA); + } else { + repository = Entity.getEntityTimeSeriesRepository(entityType); + } + total += repository.getTimeSeriesDao().listCount(listFilter); } } return total; diff --git a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json index a373325802e..5ca7d2ec821 100644 --- a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json +++ b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json @@ -36,7 +36,8 @@ "domain", "storedProcedure", "storageService", - "dataProduct" + "dataProduct", + "testCaseResolutionStatus" ], "recreateIndex": true, "batchSize": "100", diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/EntityClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/EntityClass.ts index 5f3beb40737..98dcc2a8819 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/EntityClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/EntityClass.ts @@ -13,7 +13,7 @@ import { uuid } from '../../constants/constants'; import { CustomPropertySupportedEntityList } from '../../constants/CustomProperty.constant'; -import { ENTITY_PATH, EntityType } from '../../constants/Entity.interface'; +import { EntityType, ENTITY_PATH } from '../../constants/Entity.interface'; import { createAnnouncement as createAnnouncementUtil, createInactiveAnnouncement as createInactiveAnnouncementUtil, diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts index 2de2ea00242..5b02944bf41 100644 --- a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts +++ b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts @@ -66,6 +66,7 @@ export enum EntityType { ENTITY_REPORT_DATA = 'entityReportData', WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = 'webAnalyticEntityViewReportData', WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = 'webAnalyticUserActivityReportData', + TEST_CASE_RESOLUTION_STATUS = 'test_case_resolution_status_search_index', EVENT_SUBSCRIPTION = 'eventsubscription', } diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts b/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts index e40284d3cb3..34660de15d0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts @@ -42,6 +42,7 @@ import { SearchService } from '../generated/entity/services/searchService'; import { Team } from '../generated/entity/teams/team'; import { User } from '../generated/entity/teams/user'; import { TestCase } from '../generated/tests/testCase'; +import { TestCaseResolutionStatus } from '../generated/tests/testCaseResolutionStatus'; import { TestSuite } from '../generated/tests/testSuite'; import { TagLabel } from '../generated/type/tagLabel'; import { AggregatedCostAnalysisReportDataSearchSource } from './data-insight.interface'; @@ -131,6 +132,16 @@ export interface TestCaseSearchSource } // extends EntityInterface export interface TestSuiteSearchSource extends SearchSourceBase, TestSuite {} +export interface TestCaseResolutionStatusSearchSource + extends SearchSourceBase, + TestCaseResolutionStatus { + name: string; + displayName: string; + fullyQualifiedName: string; + serviceType: string; + description: string; +} + export interface IngestionPipelineSearchSource extends SearchSourceBase, IngestionPipeline {} diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json index b92c713ad6b..60f434f806e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json @@ -51,7 +51,8 @@ "webAnalyticUserActivityReportData", "domain", "storedProcedure", - "dataProduct" + "dataProduct", + "testCaseResolutionStatus" ] }, "default": [ @@ -89,7 +90,8 @@ "domain", "storedProcedure", "dataProduct", - "ingestionPipeline" + "ingestionPipeline", + "testCaseResolutionStatus" ], "uniqueItems": true }, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts index e6762cc6c25..359f7c8cf87 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts @@ -188,6 +188,11 @@ export const MOCK_APPLICATION_ENTITY_STATS = { failedRecords: 0, successRecords: 4, }, + [EntityType.TEST_CASE_RESOLUTION_STATUS]: { + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, }; export const MOCK_APPLICATION_ENTITY_STATS_DATA = [ @@ -401,4 +406,10 @@ export const MOCK_APPLICATION_ENTITY_STATS_DATA = [ failedRecords: 0, successRecords: 4, }, + { + name: EntityType.TEST_CASE_RESOLUTION_STATUS, + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, ];