MINOR - added test case results to search reindex (#18277)

* feat: added test case results to search reindex

* fix: failing typescript test case
This commit is contained in:
Teddy 2024-10-17 11:50:46 +02:00 committed by GitHub
parent 6006a0a67f
commit d20ee5cc8a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 109 additions and 43 deletions

View File

@ -1,6 +1,8 @@
package org.openmetadata.service.apps.bundles.searchIndex; package org.openmetadata.service.apps.bundles.searchIndex;
import static org.openmetadata.schema.system.IndexingError.ErrorSource.READER; import static org.openmetadata.schema.system.IndexingError.ErrorSource.READER;
import static org.openmetadata.service.Entity.TEST_CASE_RESOLUTION_STATUS;
import static org.openmetadata.service.Entity.TEST_CASE_RESULT;
import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS; import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS;
import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB; import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_NAME_LIST_KEY; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_NAME_LIST_KEY;
@ -90,6 +92,7 @@ public class SearchIndexApp extends AbstractNativeApplication {
"storedProcedure", "storedProcedure",
"storageService", "storageService",
"testCaseResolutionStatus", "testCaseResolutionStatus",
"testCaseResult",
"apiService", "apiService",
"apiEndpoint", "apiEndpoint",
"apiCollection", "apiCollection",
@ -101,7 +104,8 @@ public class SearchIndexApp extends AbstractNativeApplication {
ReportData.ReportDataType.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA.value(), ReportData.ReportDataType.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA.value(),
ReportData.ReportDataType.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA.value(), ReportData.ReportDataType.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA.value(),
ReportData.ReportDataType.AGGREGATED_COST_ANALYSIS_REPORT_DATA.value(), ReportData.ReportDataType.AGGREGATED_COST_ANALYSIS_REPORT_DATA.value(),
"testCaseResolutionStatus"); TEST_CASE_RESOLUTION_STATUS,
TEST_CASE_RESULT);
private final List<Source> paginatedSources = new ArrayList<>(); private final List<Source> paginatedSources = new ArrayList<>();
private Processor entityProcessor; private Processor entityProcessor;
private Processor entityTimeSeriesProcessor; private Processor entityTimeSeriesProcessor;
@ -174,6 +178,8 @@ public class SearchIndexApp extends AbstractNativeApplication {
} }
private void initializeJob() { private void initializeJob() {
List<Source> paginatedEntityTimeSeriesSources = new ArrayList<>();
// Remove any Stale Jobs // Remove any Stale Jobs
cleanUpStaleJobsFromRuns(); cleanUpStaleJobsFromRuns();
@ -205,9 +211,11 @@ public class SearchIndexApp extends AbstractNativeApplication {
if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) { if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) {
source.setCursor(jobData.getAfterCursor()); source.setCursor(jobData.getAfterCursor());
} }
paginatedSources.add(source); paginatedEntityTimeSeriesSources.add(source);
} }
}); });
// Add Time Series Sources at the End of the List to Process them last
paginatedSources.addAll(paginatedEntityTimeSeriesSources);
if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) { if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) {
this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords); this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords);
this.entityTimeSeriesProcessor = new OpenSearchEntityTimeSeriesProcessor(totalRecords); this.entityTimeSeriesProcessor = new OpenSearchEntityTimeSeriesProcessor(totalRecords);

View File

@ -4582,6 +4582,30 @@ public interface CollectionDAO {
@Bind("json") String json, @Bind("json") String json,
@Bind("incidentStateId") String incidentStateId); @Bind("incidentStateId") String incidentStateId);
@SqlQuery(
"""
SELECT dqdts1.json FROM
data_quality_data_time_series dqdts1
INNER JOIN (
SELECT tc.fqnHash
FROM entity_relationship er
INNER JOIN test_case tc ON er.toId = tc.id
where fromEntity = 'testSuite' AND toEntity = 'testCase' and fromId = :testSuiteId
) ts ON dqdts1.entityFQNHash = ts.fqnHash
LEFT JOIN data_quality_data_time_series dqdts2 ON
(dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp)
WHERE dqdts2.entityFQNHash IS NULL""")
List<String> listLastTestCaseResultsForTestSuite(@BindMap Map<String, String> params);
@SqlQuery(
"""
SELECT dqdts1.json FROM
data_quality_data_time_series dqdts1
LEFT JOIN data_quality_data_time_series dqdts2 ON
(dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp)
WHERE dqdts2.entityFQNHash IS NULL AND dqdts1.entityFQNHash = :testCaseFQN""")
String listLastTestCaseResult(@BindFQN("testCaseFQN") String testCaseFQN);
default void insert( default void insert(
String testCaseFQN, String testCaseFQN,
String extension, String extension,
@ -4597,6 +4621,10 @@ public interface CollectionDAO {
json, json,
incidentStateId != null ? incidentStateId.toString() : null); incidentStateId != null ? incidentStateId.toString() : null);
} }
default List<String> listLastTestCaseResultsForTestSuite(UUID testSuiteId) {
return listLastTestCaseResultsForTestSuite(Map.of("testSuiteId", testSuiteId.toString()));
}
} }
class EntitiesCountRowMapper implements RowMapper<EntitiesCount> { class EntitiesCountRowMapper implements RowMapper<EntitiesCount> {

View File

@ -12,6 +12,7 @@ import static org.openmetadata.service.Entity.TEST_CASE_RESULT;
import static org.openmetadata.service.Entity.TEST_DEFINITION; import static org.openmetadata.service.Entity.TEST_DEFINITION;
import static org.openmetadata.service.Entity.TEST_SUITE; import static org.openmetadata.service.Entity.TEST_SUITE;
import static org.openmetadata.service.Entity.getEntityByName; import static org.openmetadata.service.Entity.getEntityByName;
import static org.openmetadata.service.Entity.getEntityTimeSeriesRepository;
import static org.openmetadata.service.Entity.populateEntityFieldTags; import static org.openmetadata.service.Entity.populateEntityFieldTags;
import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound; import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound;
import static org.openmetadata.service.security.mask.PIIMasker.maskSampleData; import static org.openmetadata.service.security.mask.PIIMasker.maskSampleData;
@ -379,6 +380,7 @@ public class TestCaseRepository extends EntityRepository<TestCase> {
@SneakyThrows @SneakyThrows
private TestCaseResult getTestCaseResult(TestCase testCase) { private TestCaseResult getTestCaseResult(TestCase testCase) {
TestCaseResult testCaseResult;
if (testCase.getTestCaseResult() != null) { if (testCase.getTestCaseResult() != null) {
// we'll return the saved state if it exists otherwise we'll fetch it from the database // we'll return the saved state if it exists otherwise we'll fetch it from the database
// Should be the case if listing from the search repo. as the test case result // Should be the case if listing from the search repo. as the test case result
@ -387,10 +389,21 @@ public class TestCaseRepository extends EntityRepository<TestCase> {
} }
SearchListFilter searchListFilter = new SearchListFilter(); SearchListFilter searchListFilter = new SearchListFilter();
searchListFilter.addQueryParam("testCaseFQN", testCase.getFullyQualifiedName()); searchListFilter.addQueryParam("testCaseFQN", testCase.getFullyQualifiedName());
EntityTimeSeriesRepository<?> timeSeriesRepository = TestCaseResultRepository timeSeriesRepository =
Entity.getEntityTimeSeriesRepository(TEST_CASE_RESULT); (TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT);
return (TestCaseResult) try {
timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null); testCaseResult =
timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null);
} catch (Exception e) {
// Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
// to database
LOG.debug(
"Error fetching test case result from search. Fetching from test case results from database",
e);
testCaseResult =
timeSeriesRepository.listLastTestCaseResult(testCase.getFullyQualifiedName());
}
return testCaseResult;
} }
public ResultList<TestCaseResult> getTestCaseResults(String fqn, Long startTs, Long endTs) { public ResultList<TestCaseResult> getTestCaseResults(String fqn, Long startTs, Long endTs) {

View File

@ -96,6 +96,21 @@ public class TestCaseResultRepository extends EntityTimeSeriesRepository<TestCas
return Response.created(uriInfo.getRequestUri()).entity(testCaseResult).build(); return Response.created(uriInfo.getRequestUri()).entity(testCaseResult).build();
} }
public ResultList<TestCaseResult> listLastTestCaseResultsForTestSuite(UUID testSuiteId) {
List<String> json =
((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao)
.listLastTestCaseResultsForTestSuite(testSuiteId);
List<TestCaseResult> testCaseResults = JsonUtils.readObjects(json, TestCaseResult.class);
return new ResultList<>(testCaseResults, null, null, testCaseResults.size());
}
public TestCaseResult listLastTestCaseResult(String testCaseFQN) {
String json =
((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao)
.listLastTestCaseResult(testCaseFQN);
return JsonUtils.readValue(json, TestCaseResult.class);
}
@Override @Override
protected void postCreate(TestCaseResult entity) { protected void postCreate(TestCaseResult entity) {
super.postCreate(entity); super.postCreate(entity);

View File

@ -51,38 +51,6 @@ public class TestSuiteRepository extends EntityRepository<TestSuite> {
private static final String UPDATE_FIELDS = "tests"; private static final String UPDATE_FIELDS = "tests";
private static final String PATCH_FIELDS = "tests"; private static final String PATCH_FIELDS = "tests";
private static final String EXECUTION_SUMMARY_AGGS =
"""
{
"aggregations": {
"status_counts": {
"terms": {
"field": "testCaseResult.testCaseStatus"
}
}
}
}
""";
private static final String ENTITY_EXECUTION_SUMMARY_AGGS =
"""
{
"aggregations": {
"entityLinks": {
"terms": {
"field": "entityLink.nonNormalized"
},
"aggs": {
"status_counts": {
"terms": {
"field": "testCaseResult.testCaseStatus"
}
}
}
}
}
}""";
private static final String ENTITY_EXECUTION_SUMMARY_FILTER = private static final String ENTITY_EXECUTION_SUMMARY_FILTER =
""" """
{ {
@ -275,14 +243,25 @@ public class TestSuiteRepository extends EntityRepository<TestSuite> {
@SneakyThrows @SneakyThrows
private List<ResultSummary> getResultSummary(UUID testSuiteId) { private List<ResultSummary> getResultSummary(UUID testSuiteId) {
List<ResultSummary> resultSummaries = new ArrayList<>(); List<ResultSummary> resultSummaries = new ArrayList<>();
ResultList<TestCaseResult> latestTestCaseResultResults;
String groupBy = "testCaseFQN.keyword"; String groupBy = "testCaseFQN.keyword";
SearchListFilter searchListFilter = new SearchListFilter(); SearchListFilter searchListFilter = new SearchListFilter();
searchListFilter.addQueryParam("testSuiteId", testSuiteId.toString()); searchListFilter.addQueryParam("testSuiteId", testSuiteId.toString());
EntityTimeSeriesRepository<TestCaseResult> entityTimeSeriesRepository = TestCaseResultRepository entityTimeSeriesRepository =
(TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT); (TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT);
ResultList<TestCaseResult> latestTestCaseResultResults = try {
entityTimeSeriesRepository.listLatestFromSearch( latestTestCaseResultResults =
EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null); entityTimeSeriesRepository.listLatestFromSearch(
EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null);
} catch (Exception e) {
// Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
// to database
LOG.debug(
"Error fetching test case result from search. Fetching from test case results from database",
e);
latestTestCaseResultResults =
entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId);
}
latestTestCaseResultResults latestTestCaseResultResults
.getData() .getData()

View File

@ -39,6 +39,7 @@
"storedProcedure", "storedProcedure",
"dataProduct", "dataProduct",
"testCaseResolutionStatus", "testCaseResolutionStatus",
"testCaseResult",
"apiService", "apiService",
"apiEndpoint", "apiEndpoint",
"apiCollection" "apiCollection"

View File

@ -67,6 +67,7 @@ export enum EntityType {
WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = 'webAnalyticEntityViewReportData', WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = 'webAnalyticEntityViewReportData',
WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = 'webAnalyticUserActivityReportData', WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = 'webAnalyticUserActivityReportData',
TEST_CASE_RESOLUTION_STATUS = 'test_case_resolution_status_search_index', TEST_CASE_RESOLUTION_STATUS = 'test_case_resolution_status_search_index',
TEST_CASE_RESULT = 'test_case_result_search_index',
EVENT_SUBSCRIPTION = 'eventsubscription', EVENT_SUBSCRIPTION = 'eventsubscription',
LINEAGE_EDGE = 'lineageEdge', LINEAGE_EDGE = 'lineageEdge',
API_SERVICE = 'apiService', API_SERVICE = 'apiService',

View File

@ -45,7 +45,7 @@ import { PipelineService } from '../generated/entity/services/pipelineService';
import { SearchService } from '../generated/entity/services/searchService'; import { SearchService } from '../generated/entity/services/searchService';
import { Team } from '../generated/entity/teams/team'; import { Team } from '../generated/entity/teams/team';
import { User } from '../generated/entity/teams/user'; import { User } from '../generated/entity/teams/user';
import { TestCase } from '../generated/tests/testCase'; import { TestCase, TestCaseResult } from '../generated/tests/testCase';
import { TestCaseResolutionStatus } from '../generated/tests/testCaseResolutionStatus'; import { TestCaseResolutionStatus } from '../generated/tests/testCaseResolutionStatus';
import { TestSuite } from '../generated/tests/testSuite'; import { TestSuite } from '../generated/tests/testSuite';
import { TagLabel } from '../generated/type/tagLabel'; import { TagLabel } from '../generated/type/tagLabel';
@ -145,6 +145,15 @@ export interface TestCaseResolutionStatusSearchSource
serviceType: string; serviceType: string;
description: string; description: string;
} }
export interface TestCaseResultSearchSource
extends SearchSourceBase,
TestCaseResult {
name: string;
displayName: string;
fullyQualifiedName: string;
serviceType: string;
description: string;
}
export interface IngestionPipelineSearchSource export interface IngestionPipelineSearchSource
extends SearchSourceBase, extends SearchSourceBase,

View File

@ -59,6 +59,7 @@
"storedProcedure", "storedProcedure",
"dataProduct", "dataProduct",
"testCaseResolutionStatus", "testCaseResolutionStatus",
"testCaseResult",
"apiService", "apiService",
"apiEndpoint", "apiEndpoint",
"apiCollection", "apiCollection",

View File

@ -193,6 +193,11 @@ export const MOCK_APPLICATION_ENTITY_STATS = {
failedRecords: 0, failedRecords: 0,
successRecords: 4, successRecords: 4,
}, },
[EntityType.TEST_CASE_RESULT]: {
totalRecords: 4,
failedRecords: 0,
successRecords: 4,
},
}; };
export const MOCK_APPLICATION_ENTITY_STATS_DATA = [ export const MOCK_APPLICATION_ENTITY_STATS_DATA = [
@ -412,4 +417,10 @@ export const MOCK_APPLICATION_ENTITY_STATS_DATA = [
failedRecords: 0, failedRecords: 0,
successRecords: 4, successRecords: 4,
}, },
{
name: EntityType.TEST_CASE_RESULT,
totalRecords: 4,
failedRecords: 0,
successRecords: 4,
},
]; ];