mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-10-19 12:50:20 +00:00
MINOR - added test case results to search reindex (#18277)
* feat: added test case results to search reindex * fix: failing typescript test case
This commit is contained in:
parent
6006a0a67f
commit
d20ee5cc8a
@ -1,6 +1,8 @@
|
|||||||
package org.openmetadata.service.apps.bundles.searchIndex;
|
package org.openmetadata.service.apps.bundles.searchIndex;
|
||||||
|
|
||||||
import static org.openmetadata.schema.system.IndexingError.ErrorSource.READER;
|
import static org.openmetadata.schema.system.IndexingError.ErrorSource.READER;
|
||||||
|
import static org.openmetadata.service.Entity.TEST_CASE_RESOLUTION_STATUS;
|
||||||
|
import static org.openmetadata.service.Entity.TEST_CASE_RESULT;
|
||||||
import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS;
|
import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS;
|
||||||
import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB;
|
import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB;
|
||||||
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_NAME_LIST_KEY;
|
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_NAME_LIST_KEY;
|
||||||
@ -90,6 +92,7 @@ public class SearchIndexApp extends AbstractNativeApplication {
|
|||||||
"storedProcedure",
|
"storedProcedure",
|
||||||
"storageService",
|
"storageService",
|
||||||
"testCaseResolutionStatus",
|
"testCaseResolutionStatus",
|
||||||
|
"testCaseResult",
|
||||||
"apiService",
|
"apiService",
|
||||||
"apiEndpoint",
|
"apiEndpoint",
|
||||||
"apiCollection",
|
"apiCollection",
|
||||||
@ -101,7 +104,8 @@ public class SearchIndexApp extends AbstractNativeApplication {
|
|||||||
ReportData.ReportDataType.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA.value(),
|
ReportData.ReportDataType.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA.value(),
|
||||||
ReportData.ReportDataType.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA.value(),
|
ReportData.ReportDataType.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA.value(),
|
||||||
ReportData.ReportDataType.AGGREGATED_COST_ANALYSIS_REPORT_DATA.value(),
|
ReportData.ReportDataType.AGGREGATED_COST_ANALYSIS_REPORT_DATA.value(),
|
||||||
"testCaseResolutionStatus");
|
TEST_CASE_RESOLUTION_STATUS,
|
||||||
|
TEST_CASE_RESULT);
|
||||||
private final List<Source> paginatedSources = new ArrayList<>();
|
private final List<Source> paginatedSources = new ArrayList<>();
|
||||||
private Processor entityProcessor;
|
private Processor entityProcessor;
|
||||||
private Processor entityTimeSeriesProcessor;
|
private Processor entityTimeSeriesProcessor;
|
||||||
@ -174,6 +178,8 @@ public class SearchIndexApp extends AbstractNativeApplication {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void initializeJob() {
|
private void initializeJob() {
|
||||||
|
List<Source> paginatedEntityTimeSeriesSources = new ArrayList<>();
|
||||||
|
|
||||||
// Remove any Stale Jobs
|
// Remove any Stale Jobs
|
||||||
cleanUpStaleJobsFromRuns();
|
cleanUpStaleJobsFromRuns();
|
||||||
|
|
||||||
@ -205,9 +211,11 @@ public class SearchIndexApp extends AbstractNativeApplication {
|
|||||||
if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) {
|
if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) {
|
||||||
source.setCursor(jobData.getAfterCursor());
|
source.setCursor(jobData.getAfterCursor());
|
||||||
}
|
}
|
||||||
paginatedSources.add(source);
|
paginatedEntityTimeSeriesSources.add(source);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
// Add Time Series Sources at the End of the List to Process them last
|
||||||
|
paginatedSources.addAll(paginatedEntityTimeSeriesSources);
|
||||||
if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) {
|
if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) {
|
||||||
this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords);
|
this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords);
|
||||||
this.entityTimeSeriesProcessor = new OpenSearchEntityTimeSeriesProcessor(totalRecords);
|
this.entityTimeSeriesProcessor = new OpenSearchEntityTimeSeriesProcessor(totalRecords);
|
||||||
|
@ -4582,6 +4582,30 @@ public interface CollectionDAO {
|
|||||||
@Bind("json") String json,
|
@Bind("json") String json,
|
||||||
@Bind("incidentStateId") String incidentStateId);
|
@Bind("incidentStateId") String incidentStateId);
|
||||||
|
|
||||||
|
@SqlQuery(
|
||||||
|
"""
|
||||||
|
SELECT dqdts1.json FROM
|
||||||
|
data_quality_data_time_series dqdts1
|
||||||
|
INNER JOIN (
|
||||||
|
SELECT tc.fqnHash
|
||||||
|
FROM entity_relationship er
|
||||||
|
INNER JOIN test_case tc ON er.toId = tc.id
|
||||||
|
where fromEntity = 'testSuite' AND toEntity = 'testCase' and fromId = :testSuiteId
|
||||||
|
) ts ON dqdts1.entityFQNHash = ts.fqnHash
|
||||||
|
LEFT JOIN data_quality_data_time_series dqdts2 ON
|
||||||
|
(dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp)
|
||||||
|
WHERE dqdts2.entityFQNHash IS NULL""")
|
||||||
|
List<String> listLastTestCaseResultsForTestSuite(@BindMap Map<String, String> params);
|
||||||
|
|
||||||
|
@SqlQuery(
|
||||||
|
"""
|
||||||
|
SELECT dqdts1.json FROM
|
||||||
|
data_quality_data_time_series dqdts1
|
||||||
|
LEFT JOIN data_quality_data_time_series dqdts2 ON
|
||||||
|
(dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp)
|
||||||
|
WHERE dqdts2.entityFQNHash IS NULL AND dqdts1.entityFQNHash = :testCaseFQN""")
|
||||||
|
String listLastTestCaseResult(@BindFQN("testCaseFQN") String testCaseFQN);
|
||||||
|
|
||||||
default void insert(
|
default void insert(
|
||||||
String testCaseFQN,
|
String testCaseFQN,
|
||||||
String extension,
|
String extension,
|
||||||
@ -4597,6 +4621,10 @@ public interface CollectionDAO {
|
|||||||
json,
|
json,
|
||||||
incidentStateId != null ? incidentStateId.toString() : null);
|
incidentStateId != null ? incidentStateId.toString() : null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
default List<String> listLastTestCaseResultsForTestSuite(UUID testSuiteId) {
|
||||||
|
return listLastTestCaseResultsForTestSuite(Map.of("testSuiteId", testSuiteId.toString()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class EntitiesCountRowMapper implements RowMapper<EntitiesCount> {
|
class EntitiesCountRowMapper implements RowMapper<EntitiesCount> {
|
||||||
|
@ -12,6 +12,7 @@ import static org.openmetadata.service.Entity.TEST_CASE_RESULT;
|
|||||||
import static org.openmetadata.service.Entity.TEST_DEFINITION;
|
import static org.openmetadata.service.Entity.TEST_DEFINITION;
|
||||||
import static org.openmetadata.service.Entity.TEST_SUITE;
|
import static org.openmetadata.service.Entity.TEST_SUITE;
|
||||||
import static org.openmetadata.service.Entity.getEntityByName;
|
import static org.openmetadata.service.Entity.getEntityByName;
|
||||||
|
import static org.openmetadata.service.Entity.getEntityTimeSeriesRepository;
|
||||||
import static org.openmetadata.service.Entity.populateEntityFieldTags;
|
import static org.openmetadata.service.Entity.populateEntityFieldTags;
|
||||||
import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound;
|
import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound;
|
||||||
import static org.openmetadata.service.security.mask.PIIMasker.maskSampleData;
|
import static org.openmetadata.service.security.mask.PIIMasker.maskSampleData;
|
||||||
@ -379,6 +380,7 @@ public class TestCaseRepository extends EntityRepository<TestCase> {
|
|||||||
|
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
private TestCaseResult getTestCaseResult(TestCase testCase) {
|
private TestCaseResult getTestCaseResult(TestCase testCase) {
|
||||||
|
TestCaseResult testCaseResult;
|
||||||
if (testCase.getTestCaseResult() != null) {
|
if (testCase.getTestCaseResult() != null) {
|
||||||
// we'll return the saved state if it exists otherwise we'll fetch it from the database
|
// we'll return the saved state if it exists otherwise we'll fetch it from the database
|
||||||
// Should be the case if listing from the search repo. as the test case result
|
// Should be the case if listing from the search repo. as the test case result
|
||||||
@ -387,10 +389,21 @@ public class TestCaseRepository extends EntityRepository<TestCase> {
|
|||||||
}
|
}
|
||||||
SearchListFilter searchListFilter = new SearchListFilter();
|
SearchListFilter searchListFilter = new SearchListFilter();
|
||||||
searchListFilter.addQueryParam("testCaseFQN", testCase.getFullyQualifiedName());
|
searchListFilter.addQueryParam("testCaseFQN", testCase.getFullyQualifiedName());
|
||||||
EntityTimeSeriesRepository<?> timeSeriesRepository =
|
TestCaseResultRepository timeSeriesRepository =
|
||||||
Entity.getEntityTimeSeriesRepository(TEST_CASE_RESULT);
|
(TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT);
|
||||||
return (TestCaseResult)
|
try {
|
||||||
timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null);
|
testCaseResult =
|
||||||
|
timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null);
|
||||||
|
} catch (Exception e) {
|
||||||
|
// Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
|
||||||
|
// to database
|
||||||
|
LOG.debug(
|
||||||
|
"Error fetching test case result from search. Fetching from test case results from database",
|
||||||
|
e);
|
||||||
|
testCaseResult =
|
||||||
|
timeSeriesRepository.listLastTestCaseResult(testCase.getFullyQualifiedName());
|
||||||
|
}
|
||||||
|
return testCaseResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
public ResultList<TestCaseResult> getTestCaseResults(String fqn, Long startTs, Long endTs) {
|
public ResultList<TestCaseResult> getTestCaseResults(String fqn, Long startTs, Long endTs) {
|
||||||
|
@ -96,6 +96,21 @@ public class TestCaseResultRepository extends EntityTimeSeriesRepository<TestCas
|
|||||||
return Response.created(uriInfo.getRequestUri()).entity(testCaseResult).build();
|
return Response.created(uriInfo.getRequestUri()).entity(testCaseResult).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ResultList<TestCaseResult> listLastTestCaseResultsForTestSuite(UUID testSuiteId) {
|
||||||
|
List<String> json =
|
||||||
|
((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao)
|
||||||
|
.listLastTestCaseResultsForTestSuite(testSuiteId);
|
||||||
|
List<TestCaseResult> testCaseResults = JsonUtils.readObjects(json, TestCaseResult.class);
|
||||||
|
return new ResultList<>(testCaseResults, null, null, testCaseResults.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
public TestCaseResult listLastTestCaseResult(String testCaseFQN) {
|
||||||
|
String json =
|
||||||
|
((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao)
|
||||||
|
.listLastTestCaseResult(testCaseFQN);
|
||||||
|
return JsonUtils.readValue(json, TestCaseResult.class);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void postCreate(TestCaseResult entity) {
|
protected void postCreate(TestCaseResult entity) {
|
||||||
super.postCreate(entity);
|
super.postCreate(entity);
|
||||||
|
@ -51,38 +51,6 @@ public class TestSuiteRepository extends EntityRepository<TestSuite> {
|
|||||||
private static final String UPDATE_FIELDS = "tests";
|
private static final String UPDATE_FIELDS = "tests";
|
||||||
private static final String PATCH_FIELDS = "tests";
|
private static final String PATCH_FIELDS = "tests";
|
||||||
|
|
||||||
private static final String EXECUTION_SUMMARY_AGGS =
|
|
||||||
"""
|
|
||||||
{
|
|
||||||
"aggregations": {
|
|
||||||
"status_counts": {
|
|
||||||
"terms": {
|
|
||||||
"field": "testCaseResult.testCaseStatus"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
""";
|
|
||||||
|
|
||||||
private static final String ENTITY_EXECUTION_SUMMARY_AGGS =
|
|
||||||
"""
|
|
||||||
{
|
|
||||||
"aggregations": {
|
|
||||||
"entityLinks": {
|
|
||||||
"terms": {
|
|
||||||
"field": "entityLink.nonNormalized"
|
|
||||||
},
|
|
||||||
"aggs": {
|
|
||||||
"status_counts": {
|
|
||||||
"terms": {
|
|
||||||
"field": "testCaseResult.testCaseStatus"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}""";
|
|
||||||
|
|
||||||
private static final String ENTITY_EXECUTION_SUMMARY_FILTER =
|
private static final String ENTITY_EXECUTION_SUMMARY_FILTER =
|
||||||
"""
|
"""
|
||||||
{
|
{
|
||||||
@ -275,14 +243,25 @@ public class TestSuiteRepository extends EntityRepository<TestSuite> {
|
|||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
private List<ResultSummary> getResultSummary(UUID testSuiteId) {
|
private List<ResultSummary> getResultSummary(UUID testSuiteId) {
|
||||||
List<ResultSummary> resultSummaries = new ArrayList<>();
|
List<ResultSummary> resultSummaries = new ArrayList<>();
|
||||||
|
ResultList<TestCaseResult> latestTestCaseResultResults;
|
||||||
String groupBy = "testCaseFQN.keyword";
|
String groupBy = "testCaseFQN.keyword";
|
||||||
SearchListFilter searchListFilter = new SearchListFilter();
|
SearchListFilter searchListFilter = new SearchListFilter();
|
||||||
searchListFilter.addQueryParam("testSuiteId", testSuiteId.toString());
|
searchListFilter.addQueryParam("testSuiteId", testSuiteId.toString());
|
||||||
EntityTimeSeriesRepository<TestCaseResult> entityTimeSeriesRepository =
|
TestCaseResultRepository entityTimeSeriesRepository =
|
||||||
(TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT);
|
(TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT);
|
||||||
ResultList<TestCaseResult> latestTestCaseResultResults =
|
try {
|
||||||
entityTimeSeriesRepository.listLatestFromSearch(
|
latestTestCaseResultResults =
|
||||||
EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null);
|
entityTimeSeriesRepository.listLatestFromSearch(
|
||||||
|
EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null);
|
||||||
|
} catch (Exception e) {
|
||||||
|
// Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
|
||||||
|
// to database
|
||||||
|
LOG.debug(
|
||||||
|
"Error fetching test case result from search. Fetching from test case results from database",
|
||||||
|
e);
|
||||||
|
latestTestCaseResultResults =
|
||||||
|
entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId);
|
||||||
|
}
|
||||||
|
|
||||||
latestTestCaseResultResults
|
latestTestCaseResultResults
|
||||||
.getData()
|
.getData()
|
||||||
|
@ -39,6 +39,7 @@
|
|||||||
"storedProcedure",
|
"storedProcedure",
|
||||||
"dataProduct",
|
"dataProduct",
|
||||||
"testCaseResolutionStatus",
|
"testCaseResolutionStatus",
|
||||||
|
"testCaseResult",
|
||||||
"apiService",
|
"apiService",
|
||||||
"apiEndpoint",
|
"apiEndpoint",
|
||||||
"apiCollection"
|
"apiCollection"
|
||||||
|
@ -67,6 +67,7 @@ export enum EntityType {
|
|||||||
WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = 'webAnalyticEntityViewReportData',
|
WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = 'webAnalyticEntityViewReportData',
|
||||||
WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = 'webAnalyticUserActivityReportData',
|
WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = 'webAnalyticUserActivityReportData',
|
||||||
TEST_CASE_RESOLUTION_STATUS = 'test_case_resolution_status_search_index',
|
TEST_CASE_RESOLUTION_STATUS = 'test_case_resolution_status_search_index',
|
||||||
|
TEST_CASE_RESULT = 'test_case_result_search_index',
|
||||||
EVENT_SUBSCRIPTION = 'eventsubscription',
|
EVENT_SUBSCRIPTION = 'eventsubscription',
|
||||||
LINEAGE_EDGE = 'lineageEdge',
|
LINEAGE_EDGE = 'lineageEdge',
|
||||||
API_SERVICE = 'apiService',
|
API_SERVICE = 'apiService',
|
||||||
|
@ -45,7 +45,7 @@ import { PipelineService } from '../generated/entity/services/pipelineService';
|
|||||||
import { SearchService } from '../generated/entity/services/searchService';
|
import { SearchService } from '../generated/entity/services/searchService';
|
||||||
import { Team } from '../generated/entity/teams/team';
|
import { Team } from '../generated/entity/teams/team';
|
||||||
import { User } from '../generated/entity/teams/user';
|
import { User } from '../generated/entity/teams/user';
|
||||||
import { TestCase } from '../generated/tests/testCase';
|
import { TestCase, TestCaseResult } from '../generated/tests/testCase';
|
||||||
import { TestCaseResolutionStatus } from '../generated/tests/testCaseResolutionStatus';
|
import { TestCaseResolutionStatus } from '../generated/tests/testCaseResolutionStatus';
|
||||||
import { TestSuite } from '../generated/tests/testSuite';
|
import { TestSuite } from '../generated/tests/testSuite';
|
||||||
import { TagLabel } from '../generated/type/tagLabel';
|
import { TagLabel } from '../generated/type/tagLabel';
|
||||||
@ -145,6 +145,15 @@ export interface TestCaseResolutionStatusSearchSource
|
|||||||
serviceType: string;
|
serviceType: string;
|
||||||
description: string;
|
description: string;
|
||||||
}
|
}
|
||||||
|
export interface TestCaseResultSearchSource
|
||||||
|
extends SearchSourceBase,
|
||||||
|
TestCaseResult {
|
||||||
|
name: string;
|
||||||
|
displayName: string;
|
||||||
|
fullyQualifiedName: string;
|
||||||
|
serviceType: string;
|
||||||
|
description: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface IngestionPipelineSearchSource
|
export interface IngestionPipelineSearchSource
|
||||||
extends SearchSourceBase,
|
extends SearchSourceBase,
|
||||||
|
@ -59,6 +59,7 @@
|
|||||||
"storedProcedure",
|
"storedProcedure",
|
||||||
"dataProduct",
|
"dataProduct",
|
||||||
"testCaseResolutionStatus",
|
"testCaseResolutionStatus",
|
||||||
|
"testCaseResult",
|
||||||
"apiService",
|
"apiService",
|
||||||
"apiEndpoint",
|
"apiEndpoint",
|
||||||
"apiCollection",
|
"apiCollection",
|
||||||
|
@ -193,6 +193,11 @@ export const MOCK_APPLICATION_ENTITY_STATS = {
|
|||||||
failedRecords: 0,
|
failedRecords: 0,
|
||||||
successRecords: 4,
|
successRecords: 4,
|
||||||
},
|
},
|
||||||
|
[EntityType.TEST_CASE_RESULT]: {
|
||||||
|
totalRecords: 4,
|
||||||
|
failedRecords: 0,
|
||||||
|
successRecords: 4,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
export const MOCK_APPLICATION_ENTITY_STATS_DATA = [
|
export const MOCK_APPLICATION_ENTITY_STATS_DATA = [
|
||||||
@ -412,4 +417,10 @@ export const MOCK_APPLICATION_ENTITY_STATS_DATA = [
|
|||||||
failedRecords: 0,
|
failedRecords: 0,
|
||||||
successRecords: 4,
|
successRecords: 4,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: EntityType.TEST_CASE_RESULT,
|
||||||
|
totalRecords: 4,
|
||||||
|
failedRecords: 0,
|
||||||
|
successRecords: 4,
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
Loading…
x
Reference in New Issue
Block a user