From adaa8e74937eb828228e6c6d95dee9bb3fdafaa2 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Wed, 21 Sep 2022 13:33:47 -0700 Subject: [PATCH] Fix #6917: startTs & endTs filters providing paginated data (#7617) * Fix #6917: startTs & endTs filters providing paginated data * Updated ui side changes * Fixed OM tests end to end test * Fix #6917: startTs & endTs filters providing paginated data * Fix #6917: startTs & endTs filters providing paginated data Co-authored-by: Shailesh Parmar Co-authored-by: Teddy Crepineau --- .../test_suite/test_e2e_workflow.py | 13 ++- .../service/jdbi3/CollectionDAO.java | 94 ++----------------- .../service/jdbi3/PipelineRepository.java | 44 ++------- .../service/jdbi3/TableRepository.java | 88 ++++------------- .../service/jdbi3/TestCaseRepository.java | 43 ++------- .../resources/databases/TableResource.java | 63 ++----------- .../resources/dqtests/TestCaseResource.java | 29 ++---- .../resources/pipelines/PipelineResource.java | 32 +------ .../databases/TableResourceTest.java | 69 +++++++++++--- .../dqtests/TestCaseResourceTest.java | 45 +++++++-- .../pipelines/PipelineResourceTest.java | 46 ++++++--- .../component/TestSummary.tsx | 2 - .../ProfilerDashboardPage.tsx | 1 - 13 files changed, 194 insertions(+), 375 deletions(-) diff --git a/ingestion/tests/integration/test_suite/test_e2e_workflow.py b/ingestion/tests/integration/test_suite/test_e2e_workflow.py index 541a4109092..c30d0430156 100644 --- a/ingestion/tests/integration/test_suite/test_e2e_workflow.py +++ b/ingestion/tests/integration/test_suite/test_e2e_workflow.py @@ -15,6 +15,7 @@ Validate workflow e2e import os import unittest +from datetime import datetime, timedelta import sqlalchemy as sqa from sqlalchemy.orm import declarative_base @@ -241,10 +242,18 @@ class TestE2EWorkflow(unittest.TestCase): assert test_case_2 test_case_result_1 = self.metadata.client.get( - "/testCase/test_suite_service_test.test_suite_database.test_suite_database_schema.users.my_test_case/testCaseResult" + "/testCase/test_suite_service_test.test_suite_database.test_suite_database_schema.users.my_test_case/testCaseResult", + data={ + "startTs": int((datetime.now() - timedelta(days=3)).timestamp()), + "endTs": int((datetime.now() + timedelta(days=3)).timestamp()), + }, ) test_case_result_2 = self.metadata.client.get( - "/testCase/test_suite_service_test.test_suite_database.test_suite_database_schema.users.table_column_name_to_exists/testCaseResult" + "/testCase/test_suite_service_test.test_suite_database.test_suite_database_schema.users.table_column_name_to_exists/testCaseResult", + data={ + "startTs": int((datetime.now() - timedelta(days=3)).timestamp()), + "endTs": int((datetime.now() + timedelta(days=3)).timestamp()), + }, ) assert test_case_result_1 diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 104883fd382..67f03ad5a72 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -2984,93 +2984,13 @@ public interface CollectionDAO { @Bind("entityFQN") String entityFQN, @Bind("extension") String extension, @Bind("timestamp") Long timestamp); @SqlQuery( - "SELECT json FROM entity_extension_time_series " - + " AND timestamp > :before ORDER BY timestamp ASC LIMIT :limit") - List listBefore( - @Define("condition") String condition, @Bind("limit") int limit, @Bind("before") long before); - - default List listBefore(ListFilter filter, int limit, long before) { - String extension = filter.getQueryParam("extension"); - String entityFQN = filter.getQueryParam("entityFQN"); - String startTs = filter.getQueryParam("startTs"); - String endTs = filter.getQueryParam("endTs"); - String condition = filter.getCondition(); - - if (extension != null) { - condition = String.format("%s AND extension='%s' ", condition, extension); - } - if (entityFQN != null) { - condition = String.format("%s AND entityFqn='%s' ", condition, entityFQN); - } - if (startTs != null && endTs != null) { - condition = - String.format( - "%s AND timestamp BETWEEN %d and %d ", condition, Long.parseLong(startTs), Long.parseLong(endTs)); - } else if (startTs != null) { - condition = String.format("%s AND timestamp > %d ", condition, Long.parseLong(startTs)); - } else if (endTs != null) { - condition = String.format("%s AND timestamp < %d ", condition, Long.parseLong(endTs)); - } - return listBefore(condition, limit, before); - } - - @SqlQuery( - "SELECT json FROM entity_extension_time_series " - + " AND timestamp < :after ORDER BY timestamp DESC LIMIT :limit") - List listAfter(@Define("condition") String condition, @Bind("limit") int limit, @Bind("after") long after); - - default List listAfter(ListFilter filter, int limit, long after) { - String extension = filter.getQueryParam("extension"); - String entityFQN = filter.getQueryParam("entityFQN"); - String startTs = filter.getQueryParam("startTs"); - String endTs = filter.getQueryParam("endTs"); - String condition = filter.getCondition(); - - if (extension != null) { - condition = String.format("%s AND extension='%s' ", condition, extension); - } - if (entityFQN != null) { - condition = String.format("%s AND entityFqn='%s' ", condition, entityFQN); - } - if (startTs != null && endTs != null) { - condition = - String.format( - "%s AND timestamp BETWEEN %d and %d ", condition, Long.parseLong(startTs), Long.parseLong(endTs)); - } else if (startTs != null) { - condition = String.format("%s AND timestamp > %d ", condition, Long.parseLong(startTs)); - } else if (endTs != null) { - condition = String.format("%s AND timestamp < %d ", condition, Long.parseLong(endTs)); - } - return listAfter(condition, limit, after); - } - - @SqlQuery("SELECT count(*) FROM entity_extension_time_series ") - int listCount(@Define("cond") String cond); - - default int listCount(ListFilter filter) { - String extension = filter.getQueryParam("extension"); - String entityFQN = filter.getQueryParam("entityFQN"); - String startTs = filter.getQueryParam("startTs"); - String endTs = filter.getQueryParam("endTs"); - String condition = filter.getCondition(); - - if (extension != null) { - condition = String.format("%s AND extension='%s' ", condition, extension); - } - if (entityFQN != null) { - condition = String.format("%s AND entityFqn='%s' ", condition, entityFQN); - } - if (startTs != null && endTs != null) { - condition = - String.format( - "%s AND timestamp BETWEEN %d and %d ", condition, Long.parseLong(startTs), Long.parseLong(endTs)); - } else if (startTs != null) { - condition = String.format("%s AND timestamp > %d ", condition, Long.parseLong(startTs)); - } else if (endTs != null) { - condition = String.format("%s AND timestamp < %d ", condition, Long.parseLong(endTs)); - } - return listCount(condition); - } + "SELECT json FROM entity_extension_time_series where entityFQN = :entityFQN and extension = :extension " + + " AND timestamp >= :startTs and timestamp <= :endTs ORDER BY timestamp DESC") + List listBetweenTimestamps( + @Bind("entityFQN") String entityFQN, + @Bind("extension") String extension, + @Bind("startTs") Long startTs, + @Bind("endTs") long endTs); } class EntitiesCountRowMapper implements RowMapper { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java index 4ae44a2c61b..8b40ab66657 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java @@ -43,7 +43,6 @@ import org.openmetadata.service.resources.pipelines.PipelineResource; import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; -import org.openmetadata.service.util.RestUtil; import org.openmetadata.service.util.ResultList; public class PipelineRepository extends EntityRepository { @@ -150,43 +149,16 @@ public class PipelineRepository extends EntityRepository { String.format("Failed to find pipeline status for %s at %s", pipeline.getName(), timestamp)); } - public ResultList getPipelineStatuses(ListFilter filter, String before, String after, int limit) - throws IOException { + public ResultList getPipelineStatuses(String fqn, Long starTs, Long endTs) throws IOException { List pipelineStatuses; - int total; - // Here timestamp is used for page marker since table profiles are sorted by timestamp - long time = Long.MAX_VALUE; + pipelineStatuses = + JsonUtils.readObjects( + daoCollection + .entityExtensionTimeSeriesDao() + .listBetweenTimestamps(fqn, PIPELINE_STATUS_EXTENSION, starTs, endTs), + PipelineStatus.class); - if (before != null) { // Reverse paging - time = Long.parseLong(RestUtil.decodeCursor(before)); - pipelineStatuses = - JsonUtils.readObjects( - daoCollection.entityExtensionTimeSeriesDao().listBefore(filter, limit + 1, time), PipelineStatus.class); - } else { // Forward paging or first page - if (after != null) { - time = Long.parseLong(RestUtil.decodeCursor(after)); - } - pipelineStatuses = - JsonUtils.readObjects( - daoCollection.entityExtensionTimeSeriesDao().listAfter(filter, limit + 1, time), PipelineStatus.class); - } - total = daoCollection.entityExtensionTimeSeriesDao().listCount(filter); - String beforeCursor = null; - String afterCursor = null; - if (before != null) { - if (pipelineStatuses.size() > limit) { // If extra result exists, then previous page exists - return before cursor - pipelineStatuses.remove(0); - beforeCursor = pipelineStatuses.get(0).getTimestamp().toString(); - } - afterCursor = pipelineStatuses.get(pipelineStatuses.size() - 1).getTimestamp().toString(); - } else { - beforeCursor = after == null ? null : pipelineStatuses.get(0).getTimestamp().toString(); - if (pipelineStatuses.size() > limit) { // If extra result exists, then next page exists - return after cursor - pipelineStatuses.remove(limit); - afterCursor = pipelineStatuses.get(limit - 1).getTimestamp().toString(); - } - } - return new ResultList<>(pipelineStatuses, beforeCursor, afterCursor, total); + return new ResultList<>(pipelineStatuses, starTs.toString(), endTs.toString(), pipelineStatuses.size()); } // Validate if a given task exists in the pipeline diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index c5511116240..91acf624677 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -754,82 +754,26 @@ public class TableRepository extends EntityRepository { } } - public ResultList getTableProfiles(ListFilter filter, String before, String after, int limit) - throws IOException { + public ResultList getTableProfiles(String fqn, Long startTs, Long endTs) throws IOException { List tableProfiles; - int total; - // Here timestamp is used for page marker since table profiles are sorted by timestamp - long time = Long.MAX_VALUE; - - if (before != null) { // Reverse paging - time = Long.parseLong(RestUtil.decodeCursor(before)); - tableProfiles = - JsonUtils.readObjects( - daoCollection.entityExtensionTimeSeriesDao().listBefore(filter, limit + 1, time), TableProfile.class); - } else { // Forward paging or first page - if (after != null) { - time = Long.parseLong(RestUtil.decodeCursor(after)); - } - tableProfiles = - JsonUtils.readObjects( - daoCollection.entityExtensionTimeSeriesDao().listAfter(filter, limit + 1, time), TableProfile.class); - } - total = daoCollection.entityExtensionTimeSeriesDao().listCount(filter); - String beforeCursor = null; - String afterCursor = null; - if (before != null) { - if (tableProfiles.size() > limit) { // If extra result exists, then previous page exists - return before cursor - tableProfiles.remove(0); - beforeCursor = tableProfiles.get(0).getTimestamp().toString(); - } - afterCursor = tableProfiles.get(tableProfiles.size() - 1).getTimestamp().toString(); - } else { - beforeCursor = after == null ? null : tableProfiles.get(0).getTimestamp().toString(); - if (tableProfiles.size() > limit) { // If extra result exists, then next page exists - return after cursor - tableProfiles.remove(limit); - afterCursor = tableProfiles.get(limit - 1).getTimestamp().toString(); - } - } - return new ResultList<>(tableProfiles, beforeCursor, afterCursor, total); + tableProfiles = + JsonUtils.readObjects( + daoCollection + .entityExtensionTimeSeriesDao() + .listBetweenTimestamps(fqn, "table.tableProfile", startTs, endTs), + TableProfile.class); + return new ResultList<>(tableProfiles, startTs.toString(), endTs.toString(), tableProfiles.size()); } - public ResultList getColumnProfiles(ListFilter filter, String before, String after, int limit) - throws IOException { + public ResultList getColumnProfiles(String fqn, Long startTs, Long endTs) throws IOException { List columnProfiles; - int total; - // Here timestamp is used for page marker since table profiles are sorted by timestamp - long time = Long.MAX_VALUE; - - if (before != null) { // Reverse paging - time = Long.parseLong(RestUtil.decodeCursor(before)); - columnProfiles = - JsonUtils.readObjects( - daoCollection.entityExtensionTimeSeriesDao().listBefore(filter, limit + 1, time), ColumnProfile.class); - } else { // Forward paging or first page - if (after != null) { - time = Long.parseLong(RestUtil.decodeCursor(after)); - } - columnProfiles = - JsonUtils.readObjects( - daoCollection.entityExtensionTimeSeriesDao().listAfter(filter, limit + 1, time), ColumnProfile.class); - } - total = daoCollection.entityExtensionTimeSeriesDao().listCount(filter); - String beforeCursor = null; - String afterCursor = null; - if (before != null) { - if (columnProfiles.size() > limit) { // If extra result exists, then previous page exists - return before cursor - columnProfiles.remove(0); - beforeCursor = columnProfiles.get(0).getTimestamp().toString(); - } - afterCursor = columnProfiles.get(columnProfiles.size() - 1).getTimestamp().toString(); - } else { - beforeCursor = after == null ? null : columnProfiles.get(0).getTimestamp().toString(); - if (columnProfiles.size() > limit) { // If extra result exists, then next page exists - return after cursor - columnProfiles.remove(limit); - afterCursor = columnProfiles.get(limit - 1).getTimestamp().toString(); - } - } - return new ResultList<>(columnProfiles, beforeCursor, afterCursor, total); + columnProfiles = + JsonUtils.readObjects( + daoCollection + .entityExtensionTimeSeriesDao() + .listBetweenTimestamps(fqn, "table.columnProfile", startTs, endTs), + ColumnProfile.class); + return new ResultList<>(columnProfiles, startTs.toString(), endTs.toString(), columnProfiles.size()); } /** diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index 719a8ec0e64..dabd38036c8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -220,43 +220,16 @@ public class TestCaseRepository extends EntityRepository { TestCaseResult.class); } - public ResultList getTestCaseResults(ListFilter filter, String before, String after, int limit) - throws IOException { + public ResultList getTestCaseResults(String fqn, Long startTs, Long endTs) throws IOException { List testCaseResults; - int total; - // Here timestamp is used for page marker since table profiles are sorted by timestamp - long time = Long.MAX_VALUE; + testCaseResults = + JsonUtils.readObjects( + daoCollection + .entityExtensionTimeSeriesDao() + .listBetweenTimestamps(fqn, TESTCASE_RESULT_EXTENSION, startTs, endTs), + TestCaseResult.class); - if (before != null) { // Reverse paging - time = Long.parseLong(RestUtil.decodeCursor(before)); - testCaseResults = - JsonUtils.readObjects( - daoCollection.entityExtensionTimeSeriesDao().listBefore(filter, limit + 1, time), TestCaseResult.class); - } else { // Forward paging or first page - if (after != null) { - time = Long.parseLong(RestUtil.decodeCursor(after)); - } - testCaseResults = - JsonUtils.readObjects( - daoCollection.entityExtensionTimeSeriesDao().listAfter(filter, limit + 1, time), TestCaseResult.class); - } - total = daoCollection.entityExtensionTimeSeriesDao().listCount(filter); - String beforeCursor = null; - String afterCursor = null; - if (before != null) { - if (testCaseResults.size() > limit) { // If extra result exists, then previous page exists - return before cursor - testCaseResults.remove(0); - beforeCursor = testCaseResults.get(0).getTimestamp().toString(); - } - afterCursor = testCaseResults.get(testCaseResults.size() - 1).getTimestamp().toString(); - } else { - beforeCursor = after == null ? null : testCaseResults.get(0).getTimestamp().toString(); - if (testCaseResults.size() > limit) { // If extra result exists, then next page exists - return after cursor - testCaseResults.remove(limit); - afterCursor = testCaseResults.get(limit - 1).getTimestamp().toString(); - } - } - return new ResultList<>(testCaseResults, beforeCursor, afterCursor, total); + return new ResultList<>(testCaseResults, String.valueOf(startTs), String.valueOf(endTs), testCaseResults.size()); } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index b587567d1ea..43de888c450 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -29,6 +29,7 @@ import javax.json.JsonPatch; import javax.validation.Valid; import javax.validation.constraints.Max; import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -70,7 +71,6 @@ import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.policyevaluator.OperationContext; import org.openmetadata.service.util.EntityUtil.Fields; -import org.openmetadata.service.util.RestUtil; import org.openmetadata.service.util.ResultList; @Path("/v1/tables") @@ -596,35 +596,10 @@ public class TableResource extends EntityResource { description = "Filter table/column profiles before the given end timestamp", schema = @Schema(type = "number")) @QueryParam("endTs") - Long endTs, - @Parameter(description = "Limit the number table profiles returned. (1 to 1000000, default = " + "10) ") - @DefaultValue("10") - @Min(0) - @Max(1000000) - @QueryParam("limit") - int limitParam, - @Parameter( - description = "Returns list of table/column profiles before this cursor", - schema = @Schema(type = "string")) - @QueryParam("before") - String before, - @Parameter( - description = "Returns list of table/column profiles after this cursor", - schema = @Schema(type = "string")) - @QueryParam("after") - String after) + Long endTs) throws IOException { - RestUtil.validateCursors(before, after); - ListFilter filter = - new ListFilter(Include.ALL).addQueryParam("entityFQN", fqn).addQueryParam("extension", "table.tableProfile"); - if (startTs != null) { - filter.addQueryParam("startTs", String.valueOf(startTs)); - } - if (endTs != null) { - filter.addQueryParam("endTs", String.valueOf(endTs)); - } - return dao.getTableProfiles(filter, before, after, limitParam); + return dao.getTableProfiles(fqn, startTs, endTs); } @GET @@ -650,41 +625,17 @@ public class TableResource extends EntityResource { @Parameter( description = "Filter table/column profiles after the given start timestamp", schema = @Schema(type = "number")) + @NotNull @QueryParam("startTs") Long startTs, @Parameter( description = "Filter table/column profiles before the given end timestamp", schema = @Schema(type = "number")) + @NotNull @QueryParam("endTs") - Long endTs, - @Parameter(description = "Limit the number table profiles returned. (1 to 1000000, default = " + "10) ") - @DefaultValue("10") - @Min(0) - @Max(1000000) - @QueryParam("limit") - int limitParam, - @Parameter( - description = "Returns list of table/column profiles before this cursor", - schema = @Schema(type = "string")) - @QueryParam("before") - String before, - @Parameter( - description = "Returns list of table/column profiles after this cursor", - schema = @Schema(type = "string")) - @QueryParam("after") - String after) + Long endTs) throws IOException { - RestUtil.validateCursors(before, after); - - ListFilter filter = - new ListFilter(Include.ALL).addQueryParam("entityFQN", fqn).addQueryParam("extension", "table.columnProfile"); - if (startTs != null) { - filter.addQueryParam("startTs", String.valueOf(startTs)); - } - if (endTs != null) { - filter.addQueryParam("endTs", String.valueOf(endTs)); - } - return dao.getColumnProfiles(filter, before, after, limitParam); + return dao.getColumnProfiles(fqn, startTs, endTs); } @PUT diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index e407c5eeae8..991f5efa455 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -33,6 +33,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.tests.CreateTestCase; import org.openmetadata.schema.tests.TestCase; @@ -465,44 +466,26 @@ public class TestCaseResource extends EntityResource listTestCaseResults( @Context SecurityContext securityContext, - @Parameter(description = "Id of the testCase", schema = @Schema(type = "string")) @PathParam("fqn") String fqn, + @Parameter(description = "fqn of the testCase", schema = @Schema(type = "string")) @PathParam("fqn") String fqn, @Parameter( description = "Filter testCase results after the given start timestamp", schema = @Schema(type = "number")) + @NonNull @QueryParam("startTs") Long startTs, @Parameter( description = "Filter testCase results before the given end timestamp", schema = @Schema(type = "number")) + @NonNull @QueryParam("endTs") - Long endTs, - @Parameter(description = "Limit the number of testCase results returned. (1 to 1000000, default = " + "10) ") - @DefaultValue("10") - @Min(0) - @Max(1000000) - @QueryParam("limit") - int limitParam, - @Parameter(description = "Returns list of testCase results before this cursor", schema = @Schema(type = "string")) - @QueryParam("before") - String before, - @Parameter(description = "Returns list of testCase results after this cursor", schema = @Schema(type = "string")) - @QueryParam("after") - String after) + Long endTs) throws IOException { - RestUtil.validateCursors(before, after); - ListFilter filter = new ListFilter(Include.ALL) .addQueryParam("entityFQN", fqn) .addQueryParam("extension", TestCaseRepository.TESTCASE_RESULT_EXTENSION); - if (startTs != null) { - filter.addQueryParam("startTs", String.valueOf(startTs)); - } - if (endTs != null) { - filter.addQueryParam("endTs", String.valueOf(endTs)); - } - return dao.getTestCaseResults(filter, before, after, limitParam); + return dao.getTestCaseResults(fqn, startTs, endTs); } @DELETE diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java index 079801e87fa..d53f0278c15 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java @@ -29,6 +29,7 @@ import javax.json.JsonPatch; import javax.validation.Valid; import javax.validation.constraints.Max; import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -392,40 +393,17 @@ public class PipelineResource extends EntityResource { Table putResponse = putTableProfileData(table.getId(), createTableProfile, authHeaders); verifyTableProfile(putResponse.getProfile(), createTableProfile.getTableProfile()); - ResultList tableProfiles = getTableProfiles(table.getFullyQualifiedName(), null, authHeaders); + ResultList tableProfiles = + getTableProfiles(table.getFullyQualifiedName(), timestamp, timestamp, authHeaders); verifyTableProfiles(tableProfiles, List.of(tableProfile), 1); ResultList tableColumnProfiles = - getColumnProfiles(table.getFullyQualifiedName() + ".c1", null, authHeaders); + getColumnProfiles( + table.getFullyQualifiedName() + ".c1", + TestUtils.dateToTimestamp("2021-09-09"), + TestUtils.dateToTimestamp("2021-09-10"), + authHeaders); verifyColumnProfiles(tableColumnProfiles, List.of(c1Profile), 1); timestamp = TestUtils.dateToTimestamp("2021-09-10"); @@ -1165,10 +1170,20 @@ public class TableResourceTest extends EntityResourceTest { putResponse = putTableProfileData(table.getId(), createTableProfile, authHeaders); verifyTableProfile(putResponse.getProfile(), createTableProfile.getTableProfile()); - tableProfiles = getTableProfiles(table.getFullyQualifiedName(), null, authHeaders); + tableProfiles = + getTableProfiles( + table.getFullyQualifiedName(), + TestUtils.dateToTimestamp("2021-09-09"), + TestUtils.dateToTimestamp("2021-09-10"), + authHeaders); verifyTableProfiles(tableProfiles, List.of(newTableProfile, tableProfile), 2); - tableColumnProfiles = getColumnProfiles(table.getFullyQualifiedName() + ".c1", null, authHeaders); + tableColumnProfiles = + getColumnProfiles( + table.getFullyQualifiedName() + ".c1", + TestUtils.dateToTimestamp("2021-09-09"), + TestUtils.dateToTimestamp("2021-09-10"), + authHeaders); verifyColumnProfiles(tableColumnProfiles, columnProfileResults, 2); // Replace table profile for a date @@ -1180,7 +1195,12 @@ public class TableResourceTest extends EntityResourceTest { table = getEntity(table.getId(), "profile", authHeaders); // first result should be the latest date - tableProfiles = getTableProfiles(table.getFullyQualifiedName(), null, authHeaders); + tableProfiles = + getTableProfiles( + table.getFullyQualifiedName(), + TestUtils.dateToTimestamp("2021-09-09"), + TestUtils.dateToTimestamp("2021-09-10"), + authHeaders); verifyTableProfiles(tableProfiles, List.of(newTableProfile1, tableProfile), 2); String dateStr = "2021-09-"; @@ -1213,11 +1233,20 @@ public class TableResourceTest extends EntityResourceTest { putTableProfileData(table.getId(), createTableProfile, authHeaders); tableProfileList.add(tableProfile); } - tableProfiles = getTableProfiles(table.getFullyQualifiedName(), tableProfileList.size(), authHeaders); + tableProfiles = + getTableProfiles( + table.getFullyQualifiedName(), + TestUtils.dateToTimestamp("2021-09-09"), + TestUtils.dateToTimestamp("2021-09-20"), + authHeaders); verifyTableProfiles(tableProfiles, tableProfileList, 12); tableColumnProfiles = - getColumnProfiles(table.getFullyQualifiedName() + ".c1", columnProfileResults.size(), authHeaders); + getColumnProfiles( + table.getFullyQualifiedName() + ".c1", + TestUtils.dateToTimestamp("2021-09-09"), + TestUtils.dateToTimestamp("2021-09-20"), + authHeaders); verifyColumnProfiles(tableColumnProfiles, columnProfileResults, 12); // Add profiles for table1 @@ -1249,11 +1278,21 @@ public class TableResourceTest extends EntityResourceTest { putTableProfileData(table1.getId(), createTableProfile, authHeaders); table1ProfileList.add(tableProfile); } - tableProfiles = getTableProfiles(table1.getFullyQualifiedName(), null, authHeaders); + tableProfiles = + getTableProfiles( + table1.getFullyQualifiedName(), + TestUtils.dateToTimestamp("2021-10-11"), + TestUtils.dateToTimestamp("2021-10-15"), + authHeaders); verifyTableProfiles(tableProfiles, table1ProfileList, 5); deleteTableProfile(table1.getFullyQualifiedName(), TABLE, TestUtils.dateToTimestamp("2021-10-11"), authHeaders); table1ProfileList.remove(0); - tableProfiles = getTableProfiles(table1.getFullyQualifiedName(), null, authHeaders); + tableProfiles = + getTableProfiles( + table1.getFullyQualifiedName(), + TestUtils.dateToTimestamp("2021-10-11"), + TestUtils.dateToTimestamp("2021-10-15"), + authHeaders); verifyTableProfiles(tableProfiles, table1ProfileList, 4); table1 = getEntity(table1.getId(), "*", authHeaders); @@ -1929,17 +1968,17 @@ public class TableResourceTest extends EntityResourceTest { TestUtils.delete(target, authHeaders); } - public static ResultList getTableProfiles(String fqn, Integer limit, Map authHeaders) - throws HttpResponseException { + public static ResultList getTableProfiles( + String fqn, Long startTs, Long endTs, Map authHeaders) throws HttpResponseException { WebTarget target = OpenMetadataApplicationTest.getResource("tables/" + fqn + "/tableProfile"); - target = limit != null ? target.queryParam("limit", limit) : target; + target = target.queryParam("startTs", startTs).queryParam("endTs", endTs); return TestUtils.get(target, TableResource.TableProfileList.class, authHeaders); } - public static ResultList getColumnProfiles(String fqn, Integer limit, Map authHeaders) - throws HttpResponseException { + public static ResultList getColumnProfiles( + String fqn, Long startTs, Long endTs, Map authHeaders) throws HttpResponseException { WebTarget target = OpenMetadataApplicationTest.getResource("tables/" + fqn + "/columnProfile"); - target = limit != null ? target.queryParam("limit", limit) : target; + target = target.queryParam("startTs", startTs).queryParam("endTs", endTs); return TestUtils.get(target, TableResource.ColumnProfileList.class, authHeaders); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index e23c5f9e961..09892799558 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -216,7 +216,11 @@ public class TestCaseResourceTest extends EntityResourceTest testCaseResults = - getTestCaseResults(testCase.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS); + getTestCaseResults( + testCase.getFullyQualifiedName(), + TestUtils.dateToTimestamp("2021-09-09"), + TestUtils.dateToTimestamp("2021-09-10"), + ADMIN_AUTH_HEADERS); verifyTestCaseResults(testCaseResults, List.of(testCaseResult), 1); // Add new date for TableCaseResult @@ -227,7 +231,12 @@ public class TestCaseResourceTest extends EntityResourceTest getTestCaseResults( - String fqn, Integer limit, Map authHeaders) throws HttpResponseException { + String fqn, Long start, Long end, Map authHeaders) throws HttpResponseException { WebTarget target = OpenMetadataApplicationTest.getResource("testCase/" + fqn + "/testCaseResult"); - target = limit != null ? target.queryParam("limit", limit) : target; + target = target.queryParam("startTs", start); + target = target.queryParam("endTs", end); return TestUtils.get(target, TestCaseResource.TestCaseResultList.class, authHeaders); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/pipelines/PipelineResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/pipelines/PipelineResourceTest.java index 30ec9f23bfc..990d43e8a56 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/pipelines/PipelineResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/pipelines/PipelineResourceTest.java @@ -294,7 +294,11 @@ public class PipelineResourceTest extends EntityResourceTest pipelineStatues = - getPipelineStatues(pipeline.getFullyQualifiedName(), null, ADMIN_AUTH_HEADERS); + getPipelineStatues( + pipeline.getFullyQualifiedName(), + TestUtils.dateToTimestamp("2022-01-15"), + TestUtils.dateToTimestamp("2022-01-16"), + ADMIN_AUTH_HEADERS); verifyPipelineStatuses(pipelineStatues, List.of(pipelineStatus), 1); // Validate that a new GET will come with the proper status @@ -311,7 +315,12 @@ public class PipelineResourceTest extends EntityResourceTest pipelineStatusList = new ArrayList<>(); - pipelineStatusList.add(pipelineStatus); - pipelineStatusList.add(newPipelineStatus1); for (int i = 11; i <= 20; i++) { pipelineStatus = new PipelineStatus() @@ -340,8 +352,12 @@ public class PipelineResourceTest extends EntityResourceTest getPipelineStatues( - String fqn, Integer limit, Map authHeaders) throws HttpResponseException { + String fqn, Long startTs, Long endTs, Map authHeaders) throws HttpResponseException { WebTarget target = OpenMetadataApplicationTest.getResource("pipelines/" + fqn + "/status"); - target = limit != null ? target.queryParam("limit", limit) : target; + target = target.queryParam("startTs", startTs).queryParam("endTs", endTs); return TestUtils.get(target, PipelineResource.PipelineStatusList.class, authHeaders); } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/TestSummary.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/TestSummary.tsx index 1890bdeda97..dbae4141a1e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/TestSummary.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ProfilerDashboard/component/TestSummary.tsx @@ -28,7 +28,6 @@ import { YAxis, } from 'recharts'; import { getListTestCaseResults } from '../../../axiosAPIs/testAPI'; -import { API_RES_MAX_SIZE } from '../../../constants/constants'; import { COLORS, PROFILER_FILTER_RANGE, @@ -135,7 +134,6 @@ const TestSummary: React.FC = ({ data }) => { { startTs, endTs, - limit: API_RES_MAX_SIZE, } ); setResults(chartData); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx index 50f2557e785..2b329128da0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ProfilerDashboardPage/ProfilerDashboardPage.tsx @@ -88,7 +88,6 @@ const ProfilerDashboardPage = () => { const { data } = await getColumnProfilerList(fqn, { startTs, endTs, - limit: API_RES_MAX_SIZE, }); setProfilerData(data || []); } catch (error) {