yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-18 12:18:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR - added test case results to search reindex (#18277)

Browse Source 
* feat: added test case results to search reindex

* fix: failing typescript test case
This commit is contained in:
Teddy 2024-10-17 11:50:46 +02:00 committed by GitHub
parent 6006a0a67f
commit d20ee5cc8a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 109 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java
View File

@ -1,6 +1,8 @@
package org.openmetadata.service.apps.bundles.searchIndex;
import static org.openmetadata.schema.system.IndexingError.ErrorSource.READER;
import static org.openmetadata.service.Entity.TEST_CASE_RESOLUTION_STATUS;
import static org.openmetadata.service.Entity.TEST_CASE_RESULT;
import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS;
import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB;
import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_NAME_LIST_KEY;
@ -90,6 +92,7 @@ public class SearchIndexApp extends AbstractNativeApplication {
"storedProcedure",
"storageService",
"testCaseResolutionStatus",
"testCaseResult",
"apiService",
"apiEndpoint",
"apiCollection",
@ -101,7 +104,8 @@ public class SearchIndexApp extends AbstractNativeApplication {
ReportData.ReportDataType.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA.value(),
ReportData.ReportDataType.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA.value(),
ReportData.ReportDataType.AGGREGATED_COST_ANALYSIS_REPORT_DATA.value(),
"testCaseResolutionStatus");
TEST_CASE_RESOLUTION_STATUS,
TEST_CASE_RESULT);
private final List<Source> paginatedSources = new ArrayList<>();
private Processor entityProcessor;
private Processor entityTimeSeriesProcessor;
@ -174,6 +178,8 @@ public class SearchIndexApp extends AbstractNativeApplication {
}
private void initializeJob() {
List<Source> paginatedEntityTimeSeriesSources = new ArrayList<>();
// Remove any Stale Jobs
cleanUpStaleJobsFromRuns();
@ -205,9 +211,11 @@ public class SearchIndexApp extends AbstractNativeApplication {
if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) {
source.setCursor(jobData.getAfterCursor());
}
paginatedSources.add(source);
paginatedEntityTimeSeriesSources.add(source);
}
});
// Add Time Series Sources at the End of the List to Process them last
paginatedSources.addAll(paginatedEntityTimeSeriesSources);
if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) {
this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords);
this.entityTimeSeriesProcessor = new OpenSearchEntityTimeSeriesProcessor(totalRecords);

28
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -4582,6 +4582,30 @@ public interface CollectionDAO {
@Bind("json") String json,
@Bind("incidentStateId") String incidentStateId);
@SqlQuery(
"""
SELECT dqdts1.json FROM
data_quality_data_time_series dqdts1
INNER JOIN (
SELECT tc.fqnHash
FROM entity_relationship er
INNER JOIN test_case tc ON er.toId = tc.id
where fromEntity = 'testSuite' AND toEntity = 'testCase' and fromId = :testSuiteId
) ts ON dqdts1.entityFQNHash = ts.fqnHash
LEFT JOIN data_quality_data_time_series dqdts2 ON
(dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp)
WHERE dqdts2.entityFQNHash IS NULL""")
List<String> listLastTestCaseResultsForTestSuite(@BindMap Map<String, String> params);
@SqlQuery(
"""
SELECT dqdts1.json FROM
data_quality_data_time_series dqdts1
LEFT JOIN data_quality_data_time_series dqdts2 ON
(dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp)
WHERE dqdts2.entityFQNHash IS NULL AND dqdts1.entityFQNHash = :testCaseFQN""")
String listLastTestCaseResult(@BindFQN("testCaseFQN") String testCaseFQN);
default void insert(
String testCaseFQN,
String extension,
@ -4597,6 +4621,10 @@ public interface CollectionDAO {
json,
incidentStateId != null ? incidentStateId.toString() : null);
}
default List<String> listLastTestCaseResultsForTestSuite(UUID testSuiteId) {
return listLastTestCaseResultsForTestSuite(Map.of("testSuiteId", testSuiteId.toString()));
}
}
class EntitiesCountRowMapper implements RowMapper<EntitiesCount> {

21
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java
View File

@ -12,6 +12,7 @@ import static org.openmetadata.service.Entity.TEST_CASE_RESULT;
import static org.openmetadata.service.Entity.TEST_DEFINITION;
import static org.openmetadata.service.Entity.TEST_SUITE;
import static org.openmetadata.service.Entity.getEntityByName;
import static org.openmetadata.service.Entity.getEntityTimeSeriesRepository;
import static org.openmetadata.service.Entity.populateEntityFieldTags;
import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound;
import static org.openmetadata.service.security.mask.PIIMasker.maskSampleData;
@ -379,6 +380,7 @@ public class TestCaseRepository extends EntityRepository<TestCase> {
@SneakyThrows
private TestCaseResult getTestCaseResult(TestCase testCase) {
TestCaseResult testCaseResult;
if (testCase.getTestCaseResult() != null) {
// we'll return the saved state if it exists otherwise we'll fetch it from the database
// Should be the case if listing from the search repo. as the test case result
@ -387,10 +389,21 @@ public class TestCaseRepository extends EntityRepository<TestCase> {
}
SearchListFilter searchListFilter = new SearchListFilter();
searchListFilter.addQueryParam("testCaseFQN", testCase.getFullyQualifiedName());
EntityTimeSeriesRepository<?> timeSeriesRepository =
Entity.getEntityTimeSeriesRepository(TEST_CASE_RESULT);
return (TestCaseResult)
timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null);
TestCaseResultRepository timeSeriesRepository =
(TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT);
try {
testCaseResult =
timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null);
} catch (Exception e) {
// Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
// to database
LOG.debug(
"Error fetching test case result from search. Fetching from test case results from database",
e);
testCaseResult =
timeSeriesRepository.listLastTestCaseResult(testCase.getFullyQualifiedName());
}
return testCaseResult;
}
public ResultList<TestCaseResult> getTestCaseResults(String fqn, Long startTs, Long endTs) {

15
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java
View File

@ -96,6 +96,21 @@ public class TestCaseResultRepository extends EntityTimeSeriesRepository<TestCas
return Response.created(uriInfo.getRequestUri()).entity(testCaseResult).build();
}
public ResultList<TestCaseResult> listLastTestCaseResultsForTestSuite(UUID testSuiteId) {
List<String> json =
((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao)
.listLastTestCaseResultsForTestSuite(testSuiteId);
List<TestCaseResult> testCaseResults = JsonUtils.readObjects(json, TestCaseResult.class);
return new ResultList<>(testCaseResults, null, null, testCaseResults.size());
}
public TestCaseResult listLastTestCaseResult(String testCaseFQN) {
String json =
((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao)
.listLastTestCaseResult(testCaseFQN);
return JsonUtils.readValue(json, TestCaseResult.class);
}
@Override
protected void postCreate(TestCaseResult entity) {
super.postCreate(entity);

51
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java
View File

@ -51,38 +51,6 @@ public class TestSuiteRepository extends EntityRepository<TestSuite> {
private static final String UPDATE_FIELDS = "tests";
private static final String PATCH_FIELDS = "tests";
private static final String EXECUTION_SUMMARY_AGGS =
"""
{
"aggregations": {
"status_counts": {
"terms": {
"field": "testCaseResult.testCaseStatus"
}
}
}
}
""";
private static final String ENTITY_EXECUTION_SUMMARY_AGGS =
"""
{
"aggregations": {
"entityLinks": {
"terms": {
"field": "entityLink.nonNormalized"
},
"aggs": {
"status_counts": {
"terms": {
"field": "testCaseResult.testCaseStatus"
}
}
}
}
}
}""";
private static final String ENTITY_EXECUTION_SUMMARY_FILTER =
"""
{
@ -275,14 +243,25 @@ public class TestSuiteRepository extends EntityRepository<TestSuite> {
@SneakyThrows
private List<ResultSummary> getResultSummary(UUID testSuiteId) {
List<ResultSummary> resultSummaries = new ArrayList<>();
ResultList<TestCaseResult> latestTestCaseResultResults;
String groupBy = "testCaseFQN.keyword";
SearchListFilter searchListFilter = new SearchListFilter();
searchListFilter.addQueryParam("testSuiteId", testSuiteId.toString());
EntityTimeSeriesRepository<TestCaseResult> entityTimeSeriesRepository =
TestCaseResultRepository entityTimeSeriesRepository =
(TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT);
ResultList<TestCaseResult> latestTestCaseResultResults =
entityTimeSeriesRepository.listLatestFromSearch(
EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null);
try {
latestTestCaseResultResults =
entityTimeSeriesRepository.listLatestFromSearch(
EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null);
} catch (Exception e) {
// Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
// to database
LOG.debug(
"Error fetching test case result from search. Fetching from test case results from database",
e);
latestTestCaseResultResults =
entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId);
}
latestTestCaseResultResults
.getData()

1
openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json
View File

@ -39,6 +39,7 @@
"storedProcedure",
"dataProduct",
"testCaseResolutionStatus",
"testCaseResult",
"apiService",
"apiEndpoint",
"apiCollection"

1
openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts
View File

@ -67,6 +67,7 @@ export enum EntityType {
WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = 'webAnalyticEntityViewReportData',
WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = 'webAnalyticUserActivityReportData',
TEST_CASE_RESOLUTION_STATUS = 'test_case_resolution_status_search_index',
TEST_CASE_RESULT = 'test_case_result_search_index',
EVENT_SUBSCRIPTION = 'eventsubscription',
LINEAGE_EDGE = 'lineageEdge',
API_SERVICE = 'apiService',

11
openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts
View File

@ -45,7 +45,7 @@ import { PipelineService } from '../generated/entity/services/pipelineService';
import { SearchService } from '../generated/entity/services/searchService';
import { Team } from '../generated/entity/teams/team';
import { User } from '../generated/entity/teams/user';
import { TestCase } from '../generated/tests/testCase';
import { TestCase, TestCaseResult } from '../generated/tests/testCase';
import { TestCaseResolutionStatus } from '../generated/tests/testCaseResolutionStatus';
import { TestSuite } from '../generated/tests/testSuite';
import { TagLabel } from '../generated/type/tagLabel';
@ -145,6 +145,15 @@ export interface TestCaseResolutionStatusSearchSource
serviceType: string;
description: string;
}
export interface TestCaseResultSearchSource
extends SearchSourceBase,
TestCaseResult {
name: string;
displayName: string;
fullyQualifiedName: string;
serviceType: string;
description: string;
}
export interface IngestionPipelineSearchSource
extends SearchSourceBase,

1
openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json
View File

@ -59,6 +59,7 @@
"storedProcedure",
"dataProduct",
"testCaseResolutionStatus",
"testCaseResult",
"apiService",
"apiEndpoint",
"apiCollection",

11
openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts
View File

@ -193,6 +193,11 @@ export const MOCK_APPLICATION_ENTITY_STATS = {
failedRecords: 0,
successRecords: 4,
},
[EntityType.TEST_CASE_RESULT]: {
totalRecords: 4,
failedRecords: 0,
successRecords: 4,
},
};
export const MOCK_APPLICATION_ENTITY_STATS_DATA = [
@ -412,4 +417,10 @@ export const MOCK_APPLICATION_ENTITY_STATS_DATA = [
failedRecords: 0,
successRecords: 4,
},
{
name: EntityType.TEST_CASE_RESULT,
totalRecords: 4,
failedRecords: 0,
successRecords: 4,
},
];