yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-21 15:38:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #6917: startTs & endTs filters providing paginated data (#7617)

Browse Source 
* Fix #6917: startTs & endTs filters providing paginated data

* Updated ui side changes

* Fixed OM tests end to end test

* Fix #6917: startTs & endTs filters providing paginated data

* Fix #6917: startTs & endTs filters providing paginated data

Co-authored-by: Shailesh Parmar <shailesh.parmar.webdev@gmail.com>
Co-authored-by: Teddy Crepineau <teddy.crepineau@gmail.com>
This commit is contained in:
Sriharsha Chintalapani 2022-09-21 13:33:47 -07:00 committed by GitHub
parent e02cc31010
commit adaa8e7493
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 194 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
ingestion/tests/integration/test_suite/test_e2e_workflow.py
View File

@ -15,6 +15,7 @@ Validate workflow e2e
import os
import unittest
from datetime import datetime, timedelta
import sqlalchemy as sqa
from sqlalchemy.orm import declarative_base
@ -241,10 +242,18 @@ class TestE2EWorkflow(unittest.TestCase):
assert test_case_2
test_case_result_1 = self.metadata.client.get(
"/testCase/test_suite_service_test.test_suite_database.test_suite_database_schema.users.my_test_case/testCaseResult"
"/testCase/test_suite_service_test.test_suite_database.test_suite_database_schema.users.my_test_case/testCaseResult",
data={
"startTs": int((datetime.now() - timedelta(days=3)).timestamp()),
"endTs": int((datetime.now() + timedelta(days=3)).timestamp()),
},
)
test_case_result_2 = self.metadata.client.get(
"/testCase/test_suite_service_test.test_suite_database.test_suite_database_schema.users.table_column_name_to_exists/testCaseResult"
"/testCase/test_suite_service_test.test_suite_database.test_suite_database_schema.users.table_column_name_to_exists/testCaseResult",
data={
"startTs": int((datetime.now() - timedelta(days=3)).timestamp()),
"endTs": int((datetime.now() + timedelta(days=3)).timestamp()),
},
)
assert test_case_result_1

94
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -2984,93 +2984,13 @@ public interface CollectionDAO {
@Bind("entityFQN") String entityFQN, @Bind("extension") String extension, @Bind("timestamp") Long timestamp);
@SqlQuery(
"SELECT json FROM entity_extension_time_series <condition> "
+ " AND timestamp > :before ORDER BY timestamp ASC LIMIT :limit")
List<String> listBefore(
@Define("condition") String condition, @Bind("limit") int limit, @Bind("before") long before);
default List<String> listBefore(ListFilter filter, int limit, long before) {
String extension = filter.getQueryParam("extension");
String entityFQN = filter.getQueryParam("entityFQN");
String startTs = filter.getQueryParam("startTs");
String endTs = filter.getQueryParam("endTs");
String condition = filter.getCondition();
if (extension != null) {
condition = String.format("%s AND extension='%s' ", condition, extension);
}
if (entityFQN != null) {
condition = String.format("%s AND entityFqn='%s' ", condition, entityFQN);
}
if (startTs != null && endTs != null) {
condition =
String.format(
"%s AND timestamp BETWEEN %d and %d ", condition, Long.parseLong(startTs), Long.parseLong(endTs));
} else if (startTs != null) {
condition = String.format("%s AND timestamp > %d ", condition, Long.parseLong(startTs));
} else if (endTs != null) {
condition = String.format("%s AND timestamp < %d ", condition, Long.parseLong(endTs));
}
return listBefore(condition, limit, before);
}
@SqlQuery(
"SELECT json FROM entity_extension_time_series <condition> "
+ " AND timestamp < :after ORDER BY timestamp DESC LIMIT :limit")
List<String> listAfter(@Define("condition") String condition, @Bind("limit") int limit, @Bind("after") long after);
default List<String> listAfter(ListFilter filter, int limit, long after) {
String extension = filter.getQueryParam("extension");
String entityFQN = filter.getQueryParam("entityFQN");
String startTs = filter.getQueryParam("startTs");
String endTs = filter.getQueryParam("endTs");
String condition = filter.getCondition();
if (extension != null) {
condition = String.format("%s AND extension='%s' ", condition, extension);
}
if (entityFQN != null) {
condition = String.format("%s AND entityFqn='%s' ", condition, entityFQN);
}
if (startTs != null && endTs != null) {
condition =
String.format(
"%s AND timestamp BETWEEN %d and %d ", condition, Long.parseLong(startTs), Long.parseLong(endTs));
} else if (startTs != null) {
condition = String.format("%s AND timestamp > %d ", condition, Long.parseLong(startTs));
} else if (endTs != null) {
condition = String.format("%s AND timestamp < %d ", condition, Long.parseLong(endTs));
}
return listAfter(condition, limit, after);
}
@SqlQuery("SELECT count(*) FROM entity_extension_time_series <cond>")
int listCount(@Define("cond") String cond);
default int listCount(ListFilter filter) {
String extension = filter.getQueryParam("extension");
String entityFQN = filter.getQueryParam("entityFQN");
String startTs = filter.getQueryParam("startTs");
String endTs = filter.getQueryParam("endTs");
String condition = filter.getCondition();
if (extension != null) {
condition = String.format("%s AND extension='%s' ", condition, extension);
}
if (entityFQN != null) {
condition = String.format("%s AND entityFqn='%s' ", condition, entityFQN);
}
if (startTs != null && endTs != null) {
condition =
String.format(
"%s AND timestamp BETWEEN %d and %d ", condition, Long.parseLong(startTs), Long.parseLong(endTs));
} else if (startTs != null) {
condition = String.format("%s AND timestamp > %d ", condition, Long.parseLong(startTs));
} else if (endTs != null) {
condition = String.format("%s AND timestamp < %d ", condition, Long.parseLong(endTs));
}
return listCount(condition);
}
"SELECT json FROM entity_extension_time_series where entityFQN = :entityFQN and extension = :extension "
+ " AND timestamp >= :startTs and timestamp <= :endTs ORDER BY timestamp DESC")
List<String> listBetweenTimestamps(
@Bind("entityFQN") String entityFQN,
@Bind("extension") String extension,
@Bind("startTs") Long startTs,
@Bind("endTs") long endTs);
}
class EntitiesCountRowMapper implements RowMapper<EntitiesCount> {

44
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java
View File

@ -43,7 +43,6 @@ import org.openmetadata.service.resources.pipelines.PipelineResource;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.FullyQualifiedName;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.RestUtil;
import org.openmetadata.service.util.ResultList;
public class PipelineRepository extends EntityRepository<Pipeline> {
@ -150,43 +149,16 @@ public class PipelineRepository extends EntityRepository<Pipeline> {
String.format("Failed to find pipeline status for %s at %s", pipeline.getName(), timestamp));
}
public ResultList<PipelineStatus> getPipelineStatuses(ListFilter filter, String before, String after, int limit)
throws IOException {
public ResultList<PipelineStatus> getPipelineStatuses(String fqn, Long starTs, Long endTs) throws IOException {
List<PipelineStatus> pipelineStatuses;
int total;
// Here timestamp is used for page marker since table profiles are sorted by timestamp
long time = Long.MAX_VALUE;
pipelineStatuses =
JsonUtils.readObjects(
daoCollection
.entityExtensionTimeSeriesDao()
.listBetweenTimestamps(fqn, PIPELINE_STATUS_EXTENSION, starTs, endTs),
PipelineStatus.class);
if (before != null) { // Reverse paging
time = Long.parseLong(RestUtil.decodeCursor(before));
pipelineStatuses =
JsonUtils.readObjects(
daoCollection.entityExtensionTimeSeriesDao().listBefore(filter, limit + 1, time), PipelineStatus.class);
} else { // Forward paging or first page
if (after != null) {
time = Long.parseLong(RestUtil.decodeCursor(after));
}
pipelineStatuses =
JsonUtils.readObjects(
daoCollection.entityExtensionTimeSeriesDao().listAfter(filter, limit + 1, time), PipelineStatus.class);
}
total = daoCollection.entityExtensionTimeSeriesDao().listCount(filter);
String beforeCursor = null;
String afterCursor = null;
if (before != null) {
if (pipelineStatuses.size() > limit) { // If extra result exists, then previous page exists - return before cursor
pipelineStatuses.remove(0);
beforeCursor = pipelineStatuses.get(0).getTimestamp().toString();
}
afterCursor = pipelineStatuses.get(pipelineStatuses.size() - 1).getTimestamp().toString();
} else {
beforeCursor = after == null ? null : pipelineStatuses.get(0).getTimestamp().toString();
if (pipelineStatuses.size() > limit) { // If extra result exists, then next page exists - return after cursor
pipelineStatuses.remove(limit);
afterCursor = pipelineStatuses.get(limit - 1).getTimestamp().toString();
}
}
return new ResultList<>(pipelineStatuses, beforeCursor, afterCursor, total);
return new ResultList<>(pipelineStatuses, starTs.toString(), endTs.toString(), pipelineStatuses.size());
}
// Validate if a given task exists in the pipeline

88
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java
View File

@ -754,82 +754,26 @@ public class TableRepository extends EntityRepository<Table> {
}
}
public ResultList<TableProfile> getTableProfiles(ListFilter filter, String before, String after, int limit)
throws IOException {
public ResultList<TableProfile> getTableProfiles(String fqn, Long startTs, Long endTs) throws IOException {
List<TableProfile> tableProfiles;
int total;
// Here timestamp is used for page marker since table profiles are sorted by timestamp
long time = Long.MAX_VALUE;
if (before != null) { // Reverse paging
time = Long.parseLong(RestUtil.decodeCursor(before));
tableProfiles =
JsonUtils.readObjects(
daoCollection.entityExtensionTimeSeriesDao().listBefore(filter, limit + 1, time), TableProfile.class);
} else { // Forward paging or first page
if (after != null) {
time = Long.parseLong(RestUtil.decodeCursor(after));
}
tableProfiles =
JsonUtils.readObjects(
daoCollection.entityExtensionTimeSeriesDao().listAfter(filter, limit + 1, time), TableProfile.class);
}
total = daoCollection.entityExtensionTimeSeriesDao().listCount(filter);
String beforeCursor = null;
String afterCursor = null;
if (before != null) {
if (tableProfiles.size() > limit) { // If extra result exists, then previous page exists - return before cursor
tableProfiles.remove(0);
beforeCursor = tableProfiles.get(0).getTimestamp().toString();
}
afterCursor = tableProfiles.get(tableProfiles.size() - 1).getTimestamp().toString();
} else {
beforeCursor = after == null ? null : tableProfiles.get(0).getTimestamp().toString();
if (tableProfiles.size() > limit) { // If extra result exists, then next page exists - return after cursor
tableProfiles.remove(limit);
afterCursor = tableProfiles.get(limit - 1).getTimestamp().toString();
}
}
return new ResultList<>(tableProfiles, beforeCursor, afterCursor, total);
tableProfiles =
JsonUtils.readObjects(
daoCollection
.entityExtensionTimeSeriesDao()
.listBetweenTimestamps(fqn, "table.tableProfile", startTs, endTs),
TableProfile.class);
return new ResultList<>(tableProfiles, startTs.toString(), endTs.toString(), tableProfiles.size());
}
public ResultList<ColumnProfile> getColumnProfiles(ListFilter filter, String before, String after, int limit)
throws IOException {
public ResultList<ColumnProfile> getColumnProfiles(String fqn, Long startTs, Long endTs) throws IOException {
List<ColumnProfile> columnProfiles;
int total;
// Here timestamp is used for page marker since table profiles are sorted by timestamp
long time = Long.MAX_VALUE;
if (before != null) { // Reverse paging
time = Long.parseLong(RestUtil.decodeCursor(before));
columnProfiles =
JsonUtils.readObjects(
daoCollection.entityExtensionTimeSeriesDao().listBefore(filter, limit + 1, time), ColumnProfile.class);
} else { // Forward paging or first page
if (after != null) {
time = Long.parseLong(RestUtil.decodeCursor(after));
}
columnProfiles =
JsonUtils.readObjects(
daoCollection.entityExtensionTimeSeriesDao().listAfter(filter, limit + 1, time), ColumnProfile.class);
}
total = daoCollection.entityExtensionTimeSeriesDao().listCount(filter);
String beforeCursor = null;
String afterCursor = null;
if (before != null) {
if (columnProfiles.size() > limit) { // If extra result exists, then previous page exists - return before cursor
columnProfiles.remove(0);
beforeCursor = columnProfiles.get(0).getTimestamp().toString();
}
afterCursor = columnProfiles.get(columnProfiles.size() - 1).getTimestamp().toString();
} else {
beforeCursor = after == null ? null : columnProfiles.get(0).getTimestamp().toString();
if (columnProfiles.size() > limit) { // If extra result exists, then next page exists - return after cursor
columnProfiles.remove(limit);
afterCursor = columnProfiles.get(limit - 1).getTimestamp().toString();
}
}
return new ResultList<>(columnProfiles, beforeCursor, afterCursor, total);
columnProfiles =
JsonUtils.readObjects(
daoCollection
.entityExtensionTimeSeriesDao()
.listBetweenTimestamps(fqn, "table.columnProfile", startTs, endTs),
ColumnProfile.class);
return new ResultList<>(columnProfiles, startTs.toString(), endTs.toString(), columnProfiles.size());
}
/**

43
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java
View File

@ -220,43 +220,16 @@ public class TestCaseRepository extends EntityRepository<TestCase> {
TestCaseResult.class);
}
public ResultList<TestCaseResult> getTestCaseResults(ListFilter filter, String before, String after, int limit)
throws IOException {
public ResultList<TestCaseResult> getTestCaseResults(String fqn, Long startTs, Long endTs) throws IOException {
List<TestCaseResult> testCaseResults;
int total;
// Here timestamp is used for page marker since table profiles are sorted by timestamp
long time = Long.MAX_VALUE;
testCaseResults =
JsonUtils.readObjects(
daoCollection
.entityExtensionTimeSeriesDao()
.listBetweenTimestamps(fqn, TESTCASE_RESULT_EXTENSION, startTs, endTs),
TestCaseResult.class);
if (before != null) { // Reverse paging
time = Long.parseLong(RestUtil.decodeCursor(before));
testCaseResults =
JsonUtils.readObjects(
daoCollection.entityExtensionTimeSeriesDao().listBefore(filter, limit + 1, time), TestCaseResult.class);
} else { // Forward paging or first page
if (after != null) {
time = Long.parseLong(RestUtil.decodeCursor(after));
}
testCaseResults =
JsonUtils.readObjects(
daoCollection.entityExtensionTimeSeriesDao().listAfter(filter, limit + 1, time), TestCaseResult.class);
}
total = daoCollection.entityExtensionTimeSeriesDao().listCount(filter);
String beforeCursor = null;
String afterCursor = null;
if (before != null) {
if (testCaseResults.size() > limit) { // If extra result exists, then previous page exists - return before cursor
testCaseResults.remove(0);
beforeCursor = testCaseResults.get(0).getTimestamp().toString();
}
afterCursor = testCaseResults.get(testCaseResults.size() - 1).getTimestamp().toString();
} else {
beforeCursor = after == null ? null : testCaseResults.get(0).getTimestamp().toString();
if (testCaseResults.size() > limit) { // If extra result exists, then next page exists - return after cursor
testCaseResults.remove(limit);
afterCursor = testCaseResults.get(limit - 1).getTimestamp().toString();
}
}
return new ResultList<>(testCaseResults, beforeCursor, afterCursor, total);
return new ResultList<>(testCaseResults, String.valueOf(startTs), String.valueOf(endTs), testCaseResults.size());
}
@Override

63
openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java
View File

@ -29,6 +29,7 @@ import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
@ -70,7 +71,6 @@ import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.RestUtil;
import org.openmetadata.service.util.ResultList;
@Path("/v1/tables")
@ -596,35 +596,10 @@ public class TableResource extends EntityResource<Table, TableRepository> {
description = "Filter table/column profiles before the given end timestamp",
schema = @Schema(type = "number"))
@QueryParam("endTs")
Long endTs,
@Parameter(description = "Limit the number table profiles returned. (1 to 1000000, default = " + "10) ")
@DefaultValue("10")
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,
@Parameter(
description = "Returns list of table/column profiles before this cursor",
schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(
description = "Returns list of table/column profiles after this cursor",
schema = @Schema(type = "string"))
@QueryParam("after")
String after)
Long endTs)
throws IOException {
RestUtil.validateCursors(before, after);
ListFilter filter =
new ListFilter(Include.ALL).addQueryParam("entityFQN", fqn).addQueryParam("extension", "table.tableProfile");
if (startTs != null) {
filter.addQueryParam("startTs", String.valueOf(startTs));
}
if (endTs != null) {
filter.addQueryParam("endTs", String.valueOf(endTs));
}
return dao.getTableProfiles(filter, before, after, limitParam);
return dao.getTableProfiles(fqn, startTs, endTs);
}
@GET
@ -650,41 +625,17 @@ public class TableResource extends EntityResource<Table, TableRepository> {
@Parameter(
description = "Filter table/column profiles after the given start timestamp",
schema = @Schema(type = "number"))
@NotNull
@QueryParam("startTs")
Long startTs,
@Parameter(
description = "Filter table/column profiles before the given end timestamp",
schema = @Schema(type = "number"))
@NotNull
@QueryParam("endTs")
Long endTs,
@Parameter(description = "Limit the number table profiles returned. (1 to 1000000, default = " + "10) ")
@DefaultValue("10")
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,
@Parameter(
description = "Returns list of table/column profiles before this cursor",
schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(
description = "Returns list of table/column profiles after this cursor",
schema = @Schema(type = "string"))
@QueryParam("after")
String after)
Long endTs)
throws IOException {
RestUtil.validateCursors(before, after);
ListFilter filter =
new ListFilter(Include.ALL).addQueryParam("entityFQN", fqn).addQueryParam("extension", "table.columnProfile");
if (startTs != null) {
filter.addQueryParam("startTs", String.valueOf(startTs));
}
if (endTs != null) {
filter.addQueryParam("endTs", String.valueOf(endTs));
}
return dao.getColumnProfiles(filter, before, after, limitParam);
return dao.getColumnProfiles(fqn, startTs, endTs);
}
@PUT

29
openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java
View File

@ -33,6 +33,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.api.tests.CreateTestCase;
import org.openmetadata.schema.tests.TestCase;
@ -465,44 +466,26 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
})
public ResultList<TestCaseResult> listTestCaseResults(
@Context SecurityContext securityContext,
@Parameter(description = "Id of the testCase", schema = @Schema(type = "string")) @PathParam("fqn") String fqn,
@Parameter(description = "fqn of the testCase", schema = @Schema(type = "string")) @PathParam("fqn") String fqn,
@Parameter(
description = "Filter testCase results after the given start timestamp",
schema = @Schema(type = "number"))
@NonNull
@QueryParam("startTs")
Long startTs,
@Parameter(
description = "Filter testCase results before the given end timestamp",
schema = @Schema(type = "number"))
@NonNull
@QueryParam("endTs")
Long endTs,
@Parameter(description = "Limit the number of testCase results returned. (1 to 1000000, default = " + "10) ")
@DefaultValue("10")
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,
@Parameter(description = "Returns list of testCase results before this cursor", schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(description = "Returns list of testCase results after this cursor", schema = @Schema(type = "string"))
@QueryParam("after")
String after)
Long endTs)
throws IOException {
RestUtil.validateCursors(before, after);
ListFilter filter =
new ListFilter(Include.ALL)
.addQueryParam("entityFQN", fqn)
.addQueryParam("extension", TestCaseRepository.TESTCASE_RESULT_EXTENSION);
if (startTs != null) {
filter.addQueryParam("startTs", String.valueOf(startTs));
}
if (endTs != null) {
filter.addQueryParam("endTs", String.valueOf(endTs));
}
return dao.getTestCaseResults(filter, before, after, limitParam);
return dao.getTestCaseResults(fqn, startTs, endTs);
}
@DELETE

32
openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java
View File

@ -29,6 +29,7 @@ import javax.json.JsonPatch;
import javax.validation.Valid;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
@ -392,40 +393,17 @@ public class PipelineResource extends EntityResource<Pipeline, PipelineRepositor
@Parameter(
description = "Filter pipeline statues after the given start timestamp",
schema = @Schema(type = "number"))
@NotNull
@QueryParam("startTs")
Long startTs,
@Parameter(
description = "Filter pipeline statues before the given end timestamp",
schema = @Schema(type = "number"))
@NotNull
@QueryParam("endTs")
Long endTs,
@Parameter(description = "Limit the number of pipeline statues returned. (1 to 1000000, default = " + "10) ")
@DefaultValue("10")
@Min(0)
@Max(1000000)
@QueryParam("limit")
int limitParam,
@Parameter(description = "Returns list of pipeline statues before this cursor", schema = @Schema(type = "string"))
@QueryParam("before")
String before,
@Parameter(description = "Returns list of pipeline statues after this cursor", schema = @Schema(type = "string"))
@QueryParam("after")
String after)
Long endTs)
throws IOException {
RestUtil.validateCursors(before, after);
ListFilter filter =
new ListFilter(Include.ALL)
.addQueryParam("entityFQN", fqn)
.addQueryParam("extension", PipelineRepository.PIPELINE_STATUS_EXTENSION);
if (startTs != null) {
filter.addQueryParam("startTs", String.valueOf(startTs));
}
if (endTs != null) {
filter.addQueryParam("endTs", String.valueOf(endTs));
}
return dao.getPipelineStatuses(filter, before, after, limitParam);
return dao.getPipelineStatuses(fqn, startTs, endTs);
}
@DELETE

69
openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java
View File

@ -1137,11 +1137,16 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
Table putResponse = putTableProfileData(table.getId(), createTableProfile, authHeaders);
verifyTableProfile(putResponse.getProfile(), createTableProfile.getTableProfile());
ResultList<TableProfile> tableProfiles = getTableProfiles(table.getFullyQualifiedName(), null, authHeaders);
ResultList<TableProfile> tableProfiles =
getTableProfiles(table.getFullyQualifiedName(), timestamp, timestamp, authHeaders);
verifyTableProfiles(tableProfiles, List.of(tableProfile), 1);
ResultList<ColumnProfile> tableColumnProfiles =
getColumnProfiles(table.getFullyQualifiedName() + ".c1", null, authHeaders);
getColumnProfiles(
table.getFullyQualifiedName() + ".c1",
TestUtils.dateToTimestamp("2021-09-09"),
TestUtils.dateToTimestamp("2021-09-10"),
authHeaders);
verifyColumnProfiles(tableColumnProfiles, List.of(c1Profile), 1);
timestamp = TestUtils.dateToTimestamp("2021-09-10");
@ -1165,10 +1170,20 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
putResponse = putTableProfileData(table.getId(), createTableProfile, authHeaders);
verifyTableProfile(putResponse.getProfile(), createTableProfile.getTableProfile());
tableProfiles = getTableProfiles(table.getFullyQualifiedName(), null, authHeaders);
tableProfiles =
getTableProfiles(
table.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-09-09"),
TestUtils.dateToTimestamp("2021-09-10"),
authHeaders);
verifyTableProfiles(tableProfiles, List.of(newTableProfile, tableProfile), 2);
tableColumnProfiles = getColumnProfiles(table.getFullyQualifiedName() + ".c1", null, authHeaders);
tableColumnProfiles =
getColumnProfiles(
table.getFullyQualifiedName() + ".c1",
TestUtils.dateToTimestamp("2021-09-09"),
TestUtils.dateToTimestamp("2021-09-10"),
authHeaders);
verifyColumnProfiles(tableColumnProfiles, columnProfileResults, 2);
// Replace table profile for a date
@ -1180,7 +1195,12 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
table = getEntity(table.getId(), "profile", authHeaders);
// first result should be the latest date
tableProfiles = getTableProfiles(table.getFullyQualifiedName(), null, authHeaders);
tableProfiles =
getTableProfiles(
table.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-09-09"),
TestUtils.dateToTimestamp("2021-09-10"),
authHeaders);
verifyTableProfiles(tableProfiles, List.of(newTableProfile1, tableProfile), 2);
String dateStr = "2021-09-";
@ -1213,11 +1233,20 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
putTableProfileData(table.getId(), createTableProfile, authHeaders);
tableProfileList.add(tableProfile);
}
tableProfiles = getTableProfiles(table.getFullyQualifiedName(), tableProfileList.size(), authHeaders);
tableProfiles =
getTableProfiles(
table.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-09-09"),
TestUtils.dateToTimestamp("2021-09-20"),
authHeaders);
verifyTableProfiles(tableProfiles, tableProfileList, 12);
tableColumnProfiles =
getColumnProfiles(table.getFullyQualifiedName() + ".c1", columnProfileResults.size(), authHeaders);
getColumnProfiles(
table.getFullyQualifiedName() + ".c1",
TestUtils.dateToTimestamp("2021-09-09"),
TestUtils.dateToTimestamp("2021-09-20"),
authHeaders);
verifyColumnProfiles(tableColumnProfiles, columnProfileResults, 12);
// Add profiles for table1
@ -1249,11 +1278,21 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
putTableProfileData(table1.getId(), createTableProfile, authHeaders);
table1ProfileList.add(tableProfile);
}
tableProfiles = getTableProfiles(table1.getFullyQualifiedName(), null, authHeaders);
tableProfiles =
getTableProfiles(
table1.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-10-11"),
TestUtils.dateToTimestamp("2021-10-15"),
authHeaders);
verifyTableProfiles(tableProfiles, table1ProfileList, 5);
deleteTableProfile(table1.getFullyQualifiedName(), TABLE, TestUtils.dateToTimestamp("2021-10-11"), authHeaders);
table1ProfileList.remove(0);
tableProfiles = getTableProfiles(table1.getFullyQualifiedName(), null, authHeaders);
tableProfiles =
getTableProfiles(
table1.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-10-11"),
TestUtils.dateToTimestamp("2021-10-15"),
authHeaders);
verifyTableProfiles(tableProfiles, table1ProfileList, 4);
table1 = getEntity(table1.getId(), "*", authHeaders);
@ -1929,17 +1968,17 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
TestUtils.delete(target, authHeaders);
}
public static ResultList<TableProfile> getTableProfiles(String fqn, Integer limit, Map<String, String> authHeaders)
throws HttpResponseException {
public static ResultList<TableProfile> getTableProfiles(
String fqn, Long startTs, Long endTs, Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = OpenMetadataApplicationTest.getResource("tables/" + fqn + "/tableProfile");
target = limit != null ? target.queryParam("limit", limit) : target;
target = target.queryParam("startTs", startTs).queryParam("endTs", endTs);
return TestUtils.get(target, TableResource.TableProfileList.class, authHeaders);
}
public static ResultList<ColumnProfile> getColumnProfiles(String fqn, Integer limit, Map<String, String> authHeaders)
throws HttpResponseException {
public static ResultList<ColumnProfile> getColumnProfiles(
String fqn, Long startTs, Long endTs, Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = OpenMetadataApplicationTest.getResource("tables/" + fqn + "/columnProfile");
target = limit != null ? target.queryParam("limit", limit) : target;
target = target.queryParam("startTs", startTs).queryParam("endTs", endTs);
return TestUtils.get(target, TableResource.ColumnProfileList.class, authHeaders);
}

45
openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java
View File

@ -216,7 +216,11 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
putTestCaseResult(testCase.getFullyQualifiedName(), testCaseResult, ADMIN_AUTH_HEADERS);
ResultList<TestCaseResult> testCaseResults =
getTestCaseResults(testCase.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS);
getTestCaseResults(
testCase.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-09-09"),
TestUtils.dateToTimestamp("2021-09-10"),
ADMIN_AUTH_HEADERS);
verifyTestCaseResults(testCaseResults, List.of(testCaseResult), 1);
// Add new date for TableCaseResult
@ -227,7 +231,12 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
.withTimestamp(TestUtils.dateToTimestamp("2021-09-10"));
putTestCaseResult(testCase.getFullyQualifiedName(), newTestCaseResult, ADMIN_AUTH_HEADERS);
testCaseResults = getTestCaseResults(testCase.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS);
testCaseResults =
getTestCaseResults(
testCase.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-09-09"),
TestUtils.dateToTimestamp("2021-09-10"),
ADMIN_AUTH_HEADERS);
verifyTestCaseResults(testCaseResults, List.of(newTestCaseResult, testCaseResult), 2);
// Replace table profile for a date
@ -240,7 +249,12 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
testCase = getEntity(testCase.getId(), "testCaseResult", ADMIN_AUTH_HEADERS);
// first result should be the latest date
testCaseResults = getTestCaseResults(testCase.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS);
testCaseResults =
getTestCaseResults(
testCase.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-09-09"),
TestUtils.dateToTimestamp("2021-09-10"),
ADMIN_AUTH_HEADERS);
verifyTestCaseResults(testCaseResults, List.of(newTestCaseResult1, testCaseResult), 2);
String dateStr = "2021-09-";
@ -257,7 +271,11 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
testCaseResultList.add(testCaseResult);
}
testCaseResults =
getTestCaseResults(testCase.getFullyQualifiedName(), testCaseResultList.size(), ADMIN_AUTH_HEADERS);
getTestCaseResults(
testCase.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-09-09"),
TestUtils.dateToTimestamp("2021-09-20"),
ADMIN_AUTH_HEADERS);
verifyTestCaseResults(testCaseResults, testCaseResultList, 12);
// create another table and add profiles
@ -275,12 +293,22 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
putTestCaseResult(testCase1.getFullyQualifiedName(), testCaseResult, ADMIN_AUTH_HEADERS);
testCase1ResultList.add(testCaseResult);
}
testCaseResults = getTestCaseResults(testCase1.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS);
testCaseResults =
getTestCaseResults(
testCase1.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-10-11"),
TestUtils.dateToTimestamp("2021-10-15"),
ADMIN_AUTH_HEADERS);
verifyTestCaseResults(testCaseResults, testCase1ResultList, 5);
deleteTestCaseResult(
testCase1.getFullyQualifiedName(), TestUtils.dateToTimestamp("2021-10-11"), ADMIN_AUTH_HEADERS);
testCase1ResultList.remove(0);
testCaseResults = getTestCaseResults(testCase1.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS);
testCaseResults =
getTestCaseResults(
testCase1.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-10-11"),
TestUtils.dateToTimestamp("2021-10-15"),
ADMIN_AUTH_HEADERS);
verifyTestCaseResults(testCaseResults, testCase1ResultList, 4);
}
@ -414,9 +442,10 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
}
public static ResultList<TestCaseResult> getTestCaseResults(
String fqn, Integer limit, Map<String, String> authHeaders) throws HttpResponseException {
String fqn, Long start, Long end, Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = OpenMetadataApplicationTest.getResource("testCase/" + fqn + "/testCaseResult");
target = limit != null ? target.queryParam("limit", limit) : target;
target = target.queryParam("startTs", start);
target = target.queryParam("endTs", end);
return TestUtils.get(target, TestCaseResource.TestCaseResultList.class, authHeaders);
}

46
openmetadata-service/src/test/java/org/openmetadata/service/resources/pipelines/PipelineResourceTest.java
View File

@ -294,7 +294,11 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline, CreatePip
verifyPipelineStatus(putResponse.getPipelineStatus(), pipelineStatus);
ResultList<PipelineStatus> pipelineStatues =
getPipelineStatues(pipeline.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS);
getPipelineStatues(
pipeline.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2022-01-15"),
TestUtils.dateToTimestamp("2022-01-16"),
ADMIN_AUTH_HEADERS);
verifyPipelineStatuses(pipelineStatues, List.of(pipelineStatus), 1);
// Validate that a new GET will come with the proper status
@ -311,7 +315,12 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline, CreatePip
putResponse = putPipelineStatusData(pipeline.getFullyQualifiedName(), newPipelineStatus, ADMIN_AUTH_HEADERS);
// Validate put response
verifyPipelineStatus(putResponse.getPipelineStatus(), newPipelineStatus);
pipelineStatues = getPipelineStatues(pipeline.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS);
pipelineStatues =
getPipelineStatues(
pipeline.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2022-01-15"),
TestUtils.dateToTimestamp("2022-01-16"),
ADMIN_AUTH_HEADERS);
verifyPipelineStatuses(pipelineStatues, List.of(pipelineStatus, newPipelineStatus), 2);
// Replace pipeline status for a date
@ -323,13 +332,16 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline, CreatePip
putResponse = putPipelineStatusData(pipeline.getFullyQualifiedName(), newPipelineStatus1, ADMIN_AUTH_HEADERS);
// Validate put response
verifyPipelineStatus(putResponse.getPipelineStatus(), newPipelineStatus1);
pipelineStatues = getPipelineStatues(pipeline.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS);
pipelineStatues =
getPipelineStatues(
pipeline.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2022-01-15"),
TestUtils.dateToTimestamp("2022-01-16"),
ADMIN_AUTH_HEADERS);
verifyPipelineStatuses(pipelineStatues, List.of(pipelineStatus, newPipelineStatus1), 2);
String dateStr = "2021-09-";
List<PipelineStatus> pipelineStatusList = new ArrayList<>();
pipelineStatusList.add(pipelineStatus);
pipelineStatusList.add(newPipelineStatus1);
for (int i = 11; i <= 20; i++) {
pipelineStatus =
new PipelineStatus()
@ -340,8 +352,12 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline, CreatePip
pipelineStatusList.add(pipelineStatus);
}
pipelineStatues =
getPipelineStatues(pipeline.getFullyQualifiedName(), pipelineStatusList.size(), ADMIN_AUTH_HEADERS);
verifyPipelineStatuses(pipelineStatues, pipelineStatusList, 12);
getPipelineStatues(
pipeline.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-09-11"),
TestUtils.dateToTimestamp("2021-09-20"),
ADMIN_AUTH_HEADERS);
verifyPipelineStatuses(pipelineStatues, pipelineStatusList, 10);
// create another table and add profiles
Pipeline pipeline1 =
@ -359,13 +375,21 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline, CreatePip
pipeline1StatusList.add(pipelineStatus);
}
pipelineStatues =
getPipelineStatues(pipeline1.getFullyQualifiedName(), pipelineStatusList.size(), ADMIN_AUTH_HEADERS);
getPipelineStatues(
pipeline1.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-10-11"),
TestUtils.dateToTimestamp("2021-10-15"),
ADMIN_AUTH_HEADERS);
verifyPipelineStatuses(pipelineStatues, pipeline1StatusList, 5);
deletePipelineStatus(
pipeline1.getFullyQualifiedName(), TestUtils.dateToTimestamp("2021-10-11"), ADMIN_AUTH_HEADERS);
pipeline1StatusList.remove(0);
pipelineStatues =
getPipelineStatues(pipeline1.getFullyQualifiedName(), pipelineStatusList.size(), ADMIN_AUTH_HEADERS);
getPipelineStatues(
pipeline1.getFullyQualifiedName(),
TestUtils.dateToTimestamp("2021-10-11"),
TestUtils.dateToTimestamp("2021-10-15"),
ADMIN_AUTH_HEADERS);
verifyPipelineStatuses(pipelineStatues, pipeline1StatusList, 4);
}
@ -556,9 +580,9 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline, CreatePip
}
public static ResultList<PipelineStatus> getPipelineStatues(
String fqn, Integer limit, Map<String, String> authHeaders) throws HttpResponseException {
String fqn, Long startTs, Long endTs, Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = OpenMetadataApplicationTest.getResource("pipelines/" + fqn + "/status");
target = limit != null ? target.queryParam("limit", limit) : target;
target = target.queryParam("startTs", startTs).queryParam("endTs", endTs);
return TestUtils.get(target, PipelineResource.PipelineStatusList.class, authHeaders);
}

2
openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/TestSummary.tsx
View File

@ -28,7 +28,6 @@ import {
YAxis,
} from 'recharts';
import { getListTestCaseResults } from '../../../axiosAPIs/testAPI';
import { API_RES_MAX_SIZE } from '../../../constants/constants';
import {
COLORS,
PROFILER_FILTER_RANGE,
@ -135,7 +134,6 @@ const TestSummary: React.FC<TestSummaryProps> = ({ data }) => {
{
startTs,
endTs,
limit: API_RES_MAX_SIZE,
}
);
setResults(chartData);

1
openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx
View File

@ -88,7 +88,6 @@ const ProfilerDashboardPage = () => {
const { data } = await getColumnProfilerList(fqn, {
startTs,
endTs,
limit: API_RES_MAX_SIZE,
});
setProfilerData(data || []);
} catch (error) {