yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-01 05:36:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1022 from open-metadata/issue1021

Browse Source 
Fixes #1021 - Move listing and pagination tests to common implementation
This commit is contained in:
Suresh Srinivas 2021-11-01 10:38:45 -07:00 committed by GitHub
commit c39d2024fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 199 additions and 889 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java
View File

@ -3,6 +3,7 @@ package org.openmetadata.catalog.resources;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.openmetadata.catalog.CatalogApplicationTest;
@ -32,9 +33,12 @@ import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.catalog.util.TestUtils;
import org.openmetadata.catalog.util.TestUtils.UpdateType;
import org.openmetadata.common.utils.JsonSchemaUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.json.JsonPatch;
import javax.ws.rs.client.WebTarget;
@ -47,8 +51,11 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.UUID;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static javax.ws.rs.core.Response.Status.CREATED;
import static javax.ws.rs.core.Response.Status.NOT_FOUND;
import static javax.ws.rs.core.Response.Status.OK;
@ -64,13 +71,16 @@ import static org.openmetadata.catalog.util.TestUtils.NON_EXISTENT_ENTITY;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.NO_CHANGE;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.assertEntityPagination;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
import static org.openmetadata.catalog.util.TestUtils.checkUserFollowing;
import static org.openmetadata.catalog.util.TestUtils.userAuthHeaders;
public abstract class EntityResourceTest<T> extends CatalogApplicationTest {
private static final Logger LOG = LoggerFactory.getLogger(EntityResourceTest.class);
private final Class<T> entityClass;
private final Class<? extends ResultList<T>> entityListClass;
private final String collectionName;
private final String allFields;
private final boolean supportsFollowers;
@ -95,8 +105,10 @@ public abstract class EntityResourceTest<T> extends CatalogApplicationTest {
public static final TagLabel TIER1_TAG_LABEL = new TagLabel().withTagFQN("Tier.Tier1");
public static final TagLabel TIER2_TAG_LABEL = new TagLabel().withTagFQN("Tier.Tier2");
public EntityResourceTest(Class<T> entityClass, String collectionName, String fields, boolean supportsFollowers) {
public EntityResourceTest(Class<T> entityClass, Class<? extends ResultList<T>> entityListClass, String collectionName,
String fields, boolean supportsFollowers) {
this.entityClass = entityClass;
this.entityListClass = entityListClass;
this.collectionName = collectionName;
this.allFields = fields;
this.supportsFollowers = supportsFollowers;
@ -159,7 +171,13 @@ public abstract class EntityResourceTest<T> extends CatalogApplicationTest {
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Methods to be overridden entity test class
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
public abstract Object createRequest(TestInfo test, String description, String displayName, EntityReference owner) throws URISyntaxException;
public final Object createRequest(TestInfo test, String description, String displayName, EntityReference owner)
throws URISyntaxException {
return createRequest(test, 0, description, displayName, owner);
}
public abstract Object createRequest(TestInfo test, int index, String description, String displayName,
EntityReference owner) throws URISyntaxException;
public abstract void validateCreatedEntity(T createdEntity, Object request, Map<String, String> authHeaders)
throws HttpResponseException;
@ -179,6 +197,88 @@ public abstract class EntityResourceTest<T> extends CatalogApplicationTest {
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Common entity tests for PUT operations
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@Test
public void get_entityListWithPagination_200(TestInfo test) throws HttpResponseException, URISyntaxException {
// Create a large number of tables
int maxEntities = 40;
for (int i = 0; i < maxEntities; i++) {
createEntity(createRequest(test, i, null, null, null), adminAuthHeaders());
}
// List all tables and use it for checking pagination
ResultList<T> allEntities = listEntities(null, 1000000, null, null,
adminAuthHeaders());
int totalRecords = allEntities.getData().size();
printEntities(allEntities);
// List tables with limit set from 1 to maxTables size
// Each time compare the returned list with allTables list to make sure right results are returned
for (int limit = 1; limit < maxEntities; limit++) {
String after = null;
String before;
int pageCount = 0;
int indexInAllTables = 0;
ResultList<T> forwardPage;
ResultList<T> backwardPage;
do { // For each limit (or page size) - forward scroll till the end
LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
forwardPage = listEntities(null, limit, null, after, adminAuthHeaders());
after = forwardPage.getPaging().getAfter();
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allEntities.getData(), forwardPage, limit, indexInAllTables);
if (pageCount == 0) { // CASE 0 - First page is being returned. There is no before cursor
assertNull(before);
} else {
// Make sure scrolling back based on before cursor returns the correct result
backwardPage = listEntities(null, limit, before, null, adminAuthHeaders());
assertEntityPagination(allEntities.getData(), backwardPage, limit, (indexInAllTables - limit));
}
printEntities(forwardPage);
indexInAllTables += forwardPage.getData().size();
pageCount++;
} while (after != null);
// We have now reached the last page - test backward scroll till the beginning
pageCount = 0;
indexInAllTables = totalRecords - limit - forwardPage.getData().size();
do {
LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
forwardPage = listEntities(null, limit, before, null, adminAuthHeaders());
printEntities(forwardPage);
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allEntities.getData(), forwardPage, limit, indexInAllTables);
pageCount++;
indexInAllTables -= forwardPage.getData().size();
} while (before != null);
}
}
@Test
public void get_entityListWithInvalidLimit_4xx() {
// Limit must be >= 1 and <= 1000,000
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listEntities(null, -1, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listEntities(null, 0, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listEntities(null, 1000001, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be less than or equal to 1000000]");
}
@Test
public void get_entityListWithInvalidPaginationCursors_4xx() {
// Passing both before and after cursors is invalid
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listEntities(null, 1, "", "", adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "Only one of before or after query parameter allowed");
}
@Test
public void put_entityCreate_200(TestInfo test) throws IOException, URISyntaxException {
// Create a new entity with PUT
@ -602,4 +702,27 @@ public abstract class EntityResourceTest<T> extends CatalogApplicationTest {
checkUserFollowing(userId, entityId, false, authHeaders);
return getEntity;
}
public ResultList<T> listEntities(Map<String, String> queryParams, Map<String, String> authHeaders)
throws HttpResponseException {
return listEntities(queryParams, null, null, null, authHeaders);
}
public ResultList<T> listEntities(Map<String, String> queryParams, Integer limit, String before,
String after, Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = getCollection();
for (Entry<String, String> entry : Optional.ofNullable(queryParams).orElse(Collections.emptyMap()).entrySet()) {
target = target.queryParam(entry.getKey(), entry.getValue());
}
target = limit != null ? target.queryParam("limit", limit) : target;
target = before != null ? target.queryParam("before", before) : target;
target = after != null ? target.queryParam("after", after) : target;
return TestUtils.get(target, entityListClass, authHeaders);
}
private void printEntities(ResultList<T> list) {
list.getData().forEach(entity -> LOG.info("{} {}", entityClass, getEntityInterface(entity).getFullyQualifiedName()));
LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
}

113
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/charts/ChartResourceTest.java
View File

@ -38,14 +38,14 @@ import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.catalog.util.TestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.client.WebTarget;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@ -64,19 +64,17 @@ import static org.openmetadata.catalog.util.TestUtils.LONG_ENTITY_NAME;
import static org.openmetadata.catalog.util.TestUtils.NON_EXISTENT_ENTITY;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.assertEntityPagination;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
public class ChartResourceTest extends EntityResourceTest<Chart> {
private static final Logger LOG = LoggerFactory.getLogger(ChartResourceTest.class);
public static EntityReference SUPERSET_REFERENCE;
public static EntityReference LOOKER_REFERENCE;
public static final TagLabel USER_ADDRESS_TAG_LABEL = new TagLabel().withTagFQN("User.Address");
public static final TagLabel TIER_1 = new TagLabel().withTagFQN("Tier.Tier1");
public ChartResourceTest() {
super(Chart.class, "charts", ChartResource.FIELDS, true);
super(Chart.class, ChartList.class, "charts", ChartResource.FIELDS, true);
}
@BeforeAll
@ -177,94 +175,14 @@ public class ChartResourceTest extends EntityResourceTest<Chart> {
createAndCheckEntity(create(test).withService(service), adminAuthHeaders());
// List charts by filtering on service name and ensure right charts are returned in the response
ChartList list = listCharts("service", service.getName(), adminAuthHeaders());
Map<String, String> queryParams = new HashMap<>() {{put("service", service.getName());}};
ResultList<Chart> list = listEntities(queryParams, adminAuthHeaders());
for (Chart chart : list.getData()) {
assertEquals(service.getName(), chart.getService().getName());
}
}
}
@Test
public void get_chartListWithInvalidLimitOffset_4xx() {
// Limit must be >= 1 and <= 1000,000
assertResponse(() -> listCharts(null, null, -1, null, null, adminAuthHeaders()),
BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
assertResponse(() ->listCharts(null, null, 0, null, null, adminAuthHeaders()),
BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
assertResponse(() -> listCharts(null, null, 1000001, null, null, adminAuthHeaders()),
BAD_REQUEST, "[query param limit must be less than or equal to 1000000]");
}
@Test
public void get_chartListWithInvalidPaginationCursors_4xx() {
// Passing both before and after cursors is invalid
assertResponse(() -> listCharts(null, null, 1, "", "", adminAuthHeaders()),
BAD_REQUEST, "Only one of before or after query parameter allowed");
}
@Test
public void get_chartListWithValidLimitOffset_4xx(TestInfo test) throws HttpResponseException {
// Create a large number of charts
int maxCharts = 40;
for (int i = 0; i < maxCharts; i++) {
createChart(create(test, i), adminAuthHeaders());
}
// List all charts
ChartList allCharts = listCharts(null, null, 1000000, null,
null, adminAuthHeaders());
int totalRecords = allCharts.getData().size();
printCharts(allCharts);
// List limit number charts at a time at various offsets and ensure right results are returned
for (int limit = 1; limit < maxCharts; limit++) {
String after = null;
String before;
int pageCount = 0;
int indexInAllCharts = 0;
ChartList forwardPage;
ChartList backwardPage;
do { // For each limit (or page size) - forward scroll till the end
LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
forwardPage = listCharts(null, null, limit, null, after, adminAuthHeaders());
printCharts(forwardPage);
after = forwardPage.getPaging().getAfter();
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allCharts.getData(), forwardPage, limit, indexInAllCharts);
if (pageCount == 0) { // CASE 0 - First page is being returned. There is no before cursor
assertNull(before);
} else {
// Make sure scrolling back based on before cursor returns the correct result
backwardPage = listCharts(null, null, limit, before, null, adminAuthHeaders());
assertEntityPagination(allCharts.getData(), backwardPage, limit, (indexInAllCharts - limit));
}
indexInAllCharts += forwardPage.getData().size();
pageCount++;
} while (after != null);
// We have now reached the last page - test backward scroll till the beginning
pageCount = 0;
indexInAllCharts = totalRecords - limit - forwardPage.getData().size();
do {
LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
forwardPage = listCharts(null, null, limit, before, null, adminAuthHeaders());
printCharts(forwardPage);
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allCharts.getData(), forwardPage, limit, indexInAllCharts);
pageCount++;
indexInAllCharts -= forwardPage.getData().size();
} while (before != null);
}
}
private void printCharts(ChartList list) {
list.getData().forEach(chart -> LOG.info("Chart {}", chart.getFullyQualifiedName()));
LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
@Test
public void get_nonExistentChart_404_notFound() {
@ -397,23 +315,6 @@ public class ChartResourceTest extends EntityResourceTest<Chart> {
return TestUtils.get(target, Chart.class, authHeaders);
}
public static ChartList listCharts(String fields, String serviceParam, Map<String, String> authHeaders)
throws HttpResponseException {
return listCharts(fields, serviceParam, null, null, null, authHeaders);
}
public static ChartList listCharts(String fields, String serviceParam, Integer limitParam,
String before, String after, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = getResource("charts");
target = fields != null ? target.queryParam("fields", fields) : target;
target = serviceParam != null ? target.queryParam("service", serviceParam) : target;
target = limitParam != null ? target.queryParam("limit", limitParam) : target;
target = before != null ? target.queryParam("before", before) : target;
target = after != null ? target.queryParam("after", after) : target;
return TestUtils.get(target, ChartList.class, authHeaders);
}
private void deleteChart(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
TestUtils.delete(getResource("charts/" + id), authHeaders);
@ -439,8 +340,8 @@ public class ChartResourceTest extends EntityResourceTest<Chart> {
}
@Override
public Object createRequest(TestInfo test, String description, String displayName, EntityReference owner) {
return create(test).withDescription(description).withDisplayName(displayName).withOwner(owner);
public Object createRequest(TestInfo test, int index, String description, String displayName, EntityReference owner) {
return create(test, index).withDescription(description).withDisplayName(displayName).withOwner(owner);
}
@Override

114
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java
View File

@ -41,15 +41,15 @@ import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.catalog.util.TestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.client.WebTarget;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@ -70,19 +70,17 @@ import static org.openmetadata.catalog.exception.CatalogExceptionMessage.ENTITY_
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.assertEntityPagination;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
public class DashboardResourceTest extends EntityResourceTest<Dashboard> {
private static final Logger LOG = LoggerFactory.getLogger(DashboardResourceTest.class);
public static EntityReference SUPERSET_REFERENCE;
public static EntityReference LOOKER_REFERENCE;
public static EntityReference SUPERSET_INVALID_SERVICE_REFERENCE;
public static List<EntityReference> CHART_REFERENCES;
public DashboardResourceTest() {
super(Dashboard.class, "dashboards", DashboardResource.FIELDS, true);
super(Dashboard.class, DashboardList.class, "dashboards", DashboardResource.FIELDS, true);
}
@ -209,7 +207,8 @@ public class DashboardResourceTest extends EntityResourceTest<Dashboard> {
createAndCheckEntity(create(test).withService(new EntityReference().withId(service.getId())
.withType(service.getType())), adminAuthHeaders());
// List Dashboards by filtering on service name and ensure right Dashboards are returned in the response
DashboardList list = listDashboards("service", service.getName(), adminAuthHeaders());
Map<String, String> queryParams = new HashMap<>() {{put("service", service.getName());}};
ResultList<Dashboard> list = listEntities(queryParams, adminAuthHeaders());
for (Dashboard db : list.getData()) {
assertEquals(service.getName(), db.getService().getName());
String expectedFQN = service.getName() + "." + db.getName();
@ -218,88 +217,6 @@ public class DashboardResourceTest extends EntityResourceTest<Dashboard> {
}
}
@Test
public void get_DashboardListWithInvalidLimitOffset_4xx() {
// Limit must be >= 1 and <= 1000,000
assertResponse(() -> listDashboards(null, null, -1, null, null, adminAuthHeaders()),
BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
assertResponse(() -> listDashboards(null, null, 0, null, null, adminAuthHeaders()),
BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
assertResponse(() -> listDashboards(null, null, 1000001, null, null, adminAuthHeaders()),
BAD_REQUEST, "[query param limit must be less than or equal to 1000000]");
}
@Test
public void get_DashboardListWithInvalidPaginationCursors_4xx() {
// Passing both before and after cursors is invalid
assertResponse(() -> listDashboards(null, null, 1, "", "", adminAuthHeaders()),
BAD_REQUEST, "Only one of before or after query parameter allowed");
}
@Test
public void get_DashboardListWithValidLimitOffset_4xx(TestInfo test) throws HttpResponseException {
// Create a large number of Dashboards
int maxDashboards = 40;
for (int i = 0; i < maxDashboards; i++) {
createDashboard(create(test, i), adminAuthHeaders());
}
// List all Dashboards
DashboardList allDashboards = listDashboards(null, null, 1000000, null,
null, adminAuthHeaders());
int totalRecords = allDashboards.getData().size();
printDashboards(allDashboards);
// List limit number Dashboards at a time at various offsets and ensure right results are returned
for (int limit = 1; limit < maxDashboards; limit++) {
String after = null;
String before;
int pageCount = 0;
int indexInAllDashboards = 0;
DashboardList forwardPage;
DashboardList backwardPage;
do { // For each limit (or page size) - forward scroll till the end
LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
forwardPage = listDashboards(null, null, limit, null, after, adminAuthHeaders());
printDashboards(forwardPage);
after = forwardPage.getPaging().getAfter();
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allDashboards.getData(), forwardPage, limit, indexInAllDashboards);
if (pageCount == 0) { // CASE 0 - First page is being returned. There is no before cursor
assertNull(before);
} else {
// Make sure scrolling back based on before cursor returns the correct result
backwardPage = listDashboards(null, null, limit, before, null, adminAuthHeaders());
assertEntityPagination(allDashboards.getData(), backwardPage, limit, (indexInAllDashboards - limit));
}
indexInAllDashboards += forwardPage.getData().size();
pageCount++;
} while (after != null);
// We have now reached the last page - test backward scroll till the beginning
pageCount = 0;
indexInAllDashboards = totalRecords - limit - forwardPage.getData().size();
do {
LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
forwardPage = listDashboards(null, null, limit, before, null, adminAuthHeaders());
printDashboards(forwardPage);
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allDashboards.getData(), forwardPage, limit, indexInAllDashboards);
pageCount++;
indexInAllDashboards -= forwardPage.getData().size();
} while (before != null);
}
}
private void printDashboards(DashboardList list) {
list.getData().forEach(Dashboard -> LOG.info("DB {}", Dashboard.getFullyQualifiedName()));
LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
@Test
public void put_DashboardChartsUpdate_200(TestInfo test) throws IOException {
CreateDashboard request = create(test).withService(SUPERSET_REFERENCE).withDescription(null);
@ -478,23 +395,6 @@ public class DashboardResourceTest extends EntityResourceTest<Dashboard> {
return TestUtils.get(target, Dashboard.class, authHeaders);
}
public static DashboardList listDashboards(String fields, String serviceParam, Map<String, String> authHeaders)
throws HttpResponseException {
return listDashboards(fields, serviceParam, null, null, null, authHeaders);
}
public static DashboardList listDashboards(String fields, String serviceParam, Integer limitParam,
String before, String after, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = getResource("dashboards");
target = fields != null ? target.queryParam("fields", fields): target;
target = serviceParam != null ? target.queryParam("service", serviceParam): target;
target = limitParam != null ? target.queryParam("limit", limitParam): target;
target = before != null ? target.queryParam("before", before) : target;
target = after != null ? target.queryParam("after", after) : target;
return TestUtils.get(target, DashboardList.class, authHeaders);
}
private void deleteDashboard(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
TestUtils.delete(getResource("dashboards/" + id), authHeaders);
@ -519,8 +419,8 @@ public class DashboardResourceTest extends EntityResourceTest<Dashboard> {
}
@Override
public Object createRequest(TestInfo test, String description, String displayName, EntityReference owner) {
return create(test).withDescription(description).withDisplayName(displayName).withOwner(owner);
public Object createRequest(TestInfo test, int index, String description, String displayName, EntityReference owner) {
return create(test, index).withDescription(description).withDisplayName(displayName).withOwner(owner);
}
@Override

121
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/DatabaseResourceTest.java
View File

@ -19,7 +19,6 @@ package org.openmetadata.catalog.resources.databases;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.openmetadata.catalog.Entity;
@ -33,13 +32,13 @@ import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.catalog.util.TestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.client.WebTarget;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
@ -54,15 +53,12 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.assertEntityPagination;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
public class DatabaseResourceTest extends EntityResourceTest<Database> {
private static final Logger LOG = LoggerFactory.getLogger(DatabaseResourceTest.class);
public DatabaseResourceTest() {
super(Database.class, "databases", DatabaseResource.FIELDS, false);
super(Database.class, DatabaseList.class, "databases", DatabaseResource.FIELDS, false);
}
@BeforeAll
@ -172,100 +168,14 @@ public class DatabaseResourceTest extends EntityResourceTest<Database> {
createAndCheckEntity(create(test).withService(service), adminAuthHeaders());
// List databases by filtering on service name and ensure right databases are returned in the response
DatabaseList list = listDatabases("service", service.getName(), adminAuthHeaders());
Map<String, String> queryParams = new HashMap<>(){{put("service", service.getName());}};
ResultList<Database> list = listEntities(queryParams, adminAuthHeaders());
for (Database db : list.getData()) {
assertEquals(service.getName(), db.getService().getName());
}
}
}
@Test
public void get_databaseListWithInvalidLimitOffset_4xx() {
// Limit must be >= 1 and <= 1000,000
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listDatabases(null, null, -1, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listDatabases(null, null, 0, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listDatabases(null, null, 1000001, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be less than or equal to 1000000]");
}
@Test
public void get_databaseListWithInvalidPaginationCursors_4xx() {
// Passing both before and after cursors is invalid
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listDatabases(null, null, 1, "", "", adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "Only one of before or after query parameter allowed");
}
@Order(1)
@Test
public void get_databaseListWithValidLimitOffset_4xx(TestInfo test) throws HttpResponseException {
// Create a large number of databases
int maxDatabases = 40;
for (int i = 0; i < maxDatabases; i++) {
createDatabase(create(test, i), adminAuthHeaders());
}
// List all databases
DatabaseList allDatabases = listDatabases(null, null, 1000000, null,
null, adminAuthHeaders());
int totalRecords = allDatabases.getData().size();
printDatabases(allDatabases);
// List limit number databases at a time at various offsets and ensure right results are returned
for (int limit = 1; limit < maxDatabases; limit++) {
String after = null;
String before;
int pageCount = 0;
int indexInAllDatabases = 0;
DatabaseList forwardPage;
DatabaseList backwardPage;
do { // For each limit (or page size) - forward scroll till the end
LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
forwardPage = listDatabases(null, null, limit, null, after, adminAuthHeaders());
printDatabases(forwardPage);
after = forwardPage.getPaging().getAfter();
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allDatabases.getData(), forwardPage, limit, indexInAllDatabases);
if (pageCount == 0) { // CASE 0 - First page is being returned. There is no before cursor
assertNull(before);
} else {
// Make sure scrolling back based on before cursor returns the correct result
backwardPage = listDatabases(null, null, limit, before, null, adminAuthHeaders());
assertEntityPagination(allDatabases.getData(), backwardPage, limit, (indexInAllDatabases - limit));
}
indexInAllDatabases += forwardPage.getData().size();
pageCount++;
} while (after != null);
// We have now reached the last page - test backward scroll till the beginning
pageCount = 0;
indexInAllDatabases = totalRecords - limit - forwardPage.getData().size();
do {
LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
forwardPage = listDatabases(null, null, limit, before, null, adminAuthHeaders());
printDatabases(forwardPage);
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allDatabases.getData(), forwardPage, limit, indexInAllDatabases);
pageCount++;
indexInAllDatabases -= forwardPage.getData().size();
} while (before != null);
}
}
private void printDatabases(DatabaseList list) {
list.getData().forEach(database -> LOG.info("DB {}", database.getFullyQualifiedName()));
LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
@Test
public void get_nonExistentDatabase_404_notFound() {
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
@ -408,23 +318,6 @@ public class DatabaseResourceTest extends EntityResourceTest<Database> {
return TestUtils.get(target, Database.class, authHeaders);
}
public static DatabaseList listDatabases(String fields, String serviceParam, Map<String, String> authHeaders)
throws HttpResponseException {
return listDatabases(fields, serviceParam, null, null, null, authHeaders);
}
public static DatabaseList listDatabases(String fields, String serviceParam, Integer limitParam,
String before, String after, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = getResource("databases");
target = fields != null ? target.queryParam("fields", fields): target;
target = serviceParam != null ? target.queryParam("service", serviceParam): target;
target = limitParam != null ? target.queryParam("limit", limitParam): target;
target = before != null ? target.queryParam("before", before) : target;
target = after != null ? target.queryParam("after", after) : target;
return TestUtils.get(target, DatabaseList.class, authHeaders);
}
private void deleteDatabase(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
TestUtils.delete(getResource("databases/" + id), authHeaders);
@ -450,8 +343,8 @@ public class DatabaseResourceTest extends EntityResourceTest<Database> {
}
@Override
public Object createRequest(TestInfo test, String description, String displayName, EntityReference owner) {
return create(test).withDescription(description).withOwner(owner);
public Object createRequest(TestInfo test, int index, String description, String displayName, EntityReference owner) {
return create(test, index).withDescription(description).withOwner(owner);
}
@Override

154
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java
View File

@ -55,6 +55,7 @@ import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.catalog.util.TestUtils;
import org.openmetadata.catalog.util.TestUtils.UpdateType;
import org.slf4j.Logger;
@ -99,7 +100,6 @@ import static org.openmetadata.catalog.util.TestUtils.UpdateType.MAJOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.NO_CHANGE;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.assertEntityPagination;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
import static org.openmetadata.catalog.util.TestUtils.userAuthHeaders;
@ -117,7 +117,7 @@ public class TableResourceTest extends EntityResourceTest<Table> {
public TableResourceTest() {
super(Table.class, "tables", TableResource.FIELDS, true);
super(Table.class, TableList.class, "tables", TableResource.FIELDS, true);
}
@BeforeAll
@ -827,31 +827,44 @@ public class TableResourceTest extends EntityResourceTest<Table> {
// 2*1 column tags
assertEquals(2, getTagUsageCount(USER_BANK_ACCOUNT_TAG_LABEL.getTagFQN(), adminAuthHeaders()));
TableList tableList = listTables(null, null, adminAuthHeaders()); // List tables
ResultList<Table> tableList = listEntities(null, adminAuthHeaders()); // List tables
assertEquals(2, tableList.getData().size());
assertFields(tableList.getData(), null);
// List tables with databaseFQN as filter
TableList tableList1 = listTables(null, DATABASE.getFullyQualifiedName(), adminAuthHeaders());
Map<String, String> queryParams = new HashMap<>() {{
put("database", DATABASE.getFullyQualifiedName());
}};
ResultList<Table> tableList1 = listEntities(queryParams, adminAuthHeaders());
assertEquals(tableList.getData().size(), tableList1.getData().size());
assertFields(tableList1.getData(), null);
// GET .../tables?fields=columns,tableConstraints
String fields = "columns,tableConstraints";
tableList = listTables(fields, null, adminAuthHeaders());
final String fields = "columns,tableConstraints";
queryParams = new HashMap<>() {{
put("fields", fields);
}};
tableList = listEntities(queryParams, adminAuthHeaders());
assertEquals(2, tableList.getData().size());
assertFields(tableList.getData(), fields);
// List tables with databaseFQN as filter
tableList1 = listTables(fields, DATABASE.getFullyQualifiedName(), adminAuthHeaders());
queryParams = new HashMap<>() {{
put("fields", fields);
put("database", DATABASE.getFullyQualifiedName());
}};
tableList1 = listEntities(queryParams, adminAuthHeaders());
assertEquals(tableList.getData().size(), tableList1.getData().size());
assertFields(tableList1.getData(), fields);
// GET .../tables?fields=usageSummary,owner,service
fields = "usageSummary,owner,database";
tableList = listTables(fields, null, adminAuthHeaders());
final String fields1 = "usageSummary,owner,database";
queryParams = new HashMap<>() {{
put("fields", fields1);
}};
tableList = listEntities(queryParams, adminAuthHeaders());
assertEquals(2, tableList.getData().size());
assertFields(tableList.getData(), fields);
assertFields(tableList.getData(), fields1);
for (Table table : tableList.getData()) {
assertEquals(table.getOwner().getId(), USER_OWNER1.getId());
assertEquals(table.getOwner().getType(), USER_OWNER1.getType());
@ -860,102 +873,13 @@ public class TableResourceTest extends EntityResourceTest<Table> {
}
// List tables with databaseFQN as filter
tableList1 = listTables(fields, DATABASE.getFullyQualifiedName(), adminAuthHeaders());
queryParams = new HashMap<>() {{
put("fields", fields1);
put("database", DATABASE.getFullyQualifiedName());
}};
tableList1 = listEntities(queryParams, adminAuthHeaders());
assertEquals(tableList.getData().size(), tableList1.getData().size());
assertFields(tableList1.getData(), fields);
}
@Test
public void get_tableListWithInvalidLimit_4xx() {
// Limit must be >= 1 and <= 1000,000
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listTables(null, null, -1, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listTables(null, null, 0, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listTables(null, null, 1000001, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be less than or equal to 1000000]");
}
@Test
public void get_tableListWithInvalidPaginationCursors_4xx() {
// Passing both before and after cursors is invalid
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listTables(null, null, 1, "", "", adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "Only one of before or after query parameter allowed");
}
/**
* For cursor based pagination and implementation details:
* @see org.openmetadata.catalog.util.ResultList#ResultList
*
* The tests and various CASES referenced are base on that.
*/
@Test
public void get_tableListWithPagination_200(TestInfo test) throws HttpResponseException {
// Create a large number of tables
int maxTables = 40;
for (int i = 0; i < maxTables; i++) {
createEntity(create(test, i), adminAuthHeaders());
}
// List all tables and use it for checking pagination
TableList allTables = listTables(null, null, 1000000, null, null,
adminAuthHeaders());
int totalRecords = allTables.getData().size();
printTables(allTables);
// List tables with limit set from 1 to maxTables size
// Each time compare the returned list with allTables list to make sure right results are returned
for (int limit = 1; limit < maxTables; limit++) {
String after = null;
String before;
int pageCount = 0;
int indexInAllTables = 0;
TableList forwardPage;
TableList backwardPage;
do { // For each limit (or page size) - forward scroll till the end
LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
forwardPage = listTables(null, null, limit, null, after, adminAuthHeaders());
after = forwardPage.getPaging().getAfter();
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allTables.getData(), forwardPage, limit, indexInAllTables);
if (pageCount == 0) { // CASE 0 - First page is being returned. There is no before cursor
assertNull(before);
} else {
// Make sure scrolling back based on before cursor returns the correct result
backwardPage = listTables(null, null, limit, before, null, adminAuthHeaders());
assertEntityPagination(allTables.getData(), backwardPage, limit, (indexInAllTables - limit));
}
printTables(forwardPage);
indexInAllTables += forwardPage.getData().size();
pageCount++;
} while (after != null);
// We have now reached the last page - test backward scroll till the beginning
pageCount = 0;
indexInAllTables = totalRecords - limit - forwardPage.getData().size();
do {
LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
forwardPage = listTables(null, null, limit, before, null, adminAuthHeaders());
printTables(forwardPage);
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allTables.getData(), forwardPage, limit, indexInAllTables);
pageCount++;
indexInAllTables -= forwardPage.getData().size();
} while (before != null);
}
}
private void printTables(TableList list) {
list.getData().forEach(table -> LOG.info("Table {}", table.getFullyQualifiedName()));
LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
assertFields(tableList1.getData(), fields1);
}
@Test
@ -1170,22 +1094,6 @@ public class TableResourceTest extends EntityResourceTest<Table> {
return TestUtils.get(target, Table.class, authHeaders);
}
public static TableList listTables(String fields, String databaseParam, Map<String, String> authHeaders)
throws HttpResponseException {
return listTables(fields, databaseParam, null, null, null, authHeaders);
}
public static TableList listTables(String fields, String databaseParam, Integer limit, String before, String after,
Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("tables");
target = fields != null ? target.queryParam("fields", fields) : target;
target = databaseParam != null ? target.queryParam("database", databaseParam) : target;
target = limit != null ? target.queryParam("limit", limit) : target;
target = before != null ? target.queryParam("before", before) : target;
target = after != null ? target.queryParam("after", after) : target;
return TestUtils.get(target, TableList.class, authHeaders);
}
public static CreateTable create(TestInfo test) {
return create(test, 0);
}
@ -1266,8 +1174,8 @@ public class TableResourceTest extends EntityResourceTest<Table> {
}
@Override
public Object createRequest(TestInfo test, String description, String displayName, EntityReference owner) {
return create(test).withDescription(description).withOwner(owner);
public Object createRequest(TestInfo test, int index, String description, String displayName, EntityReference owner) {
return create(test, index).withDescription(description).withOwner(owner);
}
@Override

118
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/pipelines/PipelineResourceTest.java
View File

@ -35,9 +35,8 @@ import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.type.Task;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.catalog.util.TestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response.Status;
@ -47,6 +46,7 @@ import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@ -64,18 +64,16 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.assertEntityPagination;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
public class PipelineResourceTest extends EntityResourceTest<Pipeline> {
private static final Logger LOG = LoggerFactory.getLogger(PipelineResourceTest.class);
public static List<Task> TASKS;
public static final TagLabel TIER_1 = new TagLabel().withTagFQN("Tier.Tier1");
public static final TagLabel USER_ADDRESS_TAG_LABEL = new TagLabel().withTagFQN("User.Address");
public PipelineResourceTest() {
super(Pipeline.class, "pipelines", PipelineResource.FIELDS, true);
super(Pipeline.class, PipelineList.class, "pipelines", PipelineResource.FIELDS, true);
}
@ -91,8 +89,8 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline> {
}
@Override
public Object createRequest(TestInfo test, String description, String displayName, EntityReference owner) {
return create(test).withDescription(description).withDisplayName(displayName).withOwner(owner);
public Object createRequest(TestInfo test, int index, String description, String displayName, EntityReference owner) {
return create(test, index).withDescription(description).withDisplayName(displayName).withOwner(owner);
}
@Override
@ -223,99 +221,14 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline> {
createAndCheckEntity(create(test).withService(service), adminAuthHeaders());
// List Pipelines by filtering on service name and ensure right Pipelines are returned in the response
PipelineList list = listPipelines("service", service.getName(), adminAuthHeaders());
Map<String, String> queryParams = new HashMap<>(){{put("service", service.getName());}};
ResultList<Pipeline> list = listEntities(queryParams, adminAuthHeaders());
for (Pipeline db : list.getData()) {
assertEquals(service.getName(), db.getService().getName());
}
}
}
@Test
public void get_PipelineListWithInvalidLimitOffset_4xx() {
// Limit must be >= 1 and <= 1000,000
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listPipelines(null, null, -1, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listPipelines(null, null, 0, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listPipelines(null, null, 1000001, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be less than or equal to 1000000]");
}
@Test
public void get_PipelineListWithInvalidPaginationCursors_4xx() {
// Passing both before and after cursors is invalid
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listPipelines(null, null, 1, "", "", adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "Only one of before or after query parameter allowed");
}
@Test
public void get_PipelineListWithValidLimitOffset_4xx(TestInfo test) throws HttpResponseException {
// Create a large number of Pipelines
int maxPipelines = 40;
for (int i = 0; i < maxPipelines; i++) {
createPipeline(create(test, i), adminAuthHeaders());
}
// List all Pipelines
PipelineList allPipelines = listPipelines(null, null, 1000000, null,
null, adminAuthHeaders());
int totalRecords = allPipelines.getData().size();
printPipelines(allPipelines);
// List limit number Pipelines at a time at various offsets and ensure right results are returned
for (int limit = 1; limit < maxPipelines; limit++) {
String after = null;
String before;
int pageCount = 0;
int indexInAllPipelines = 0;
PipelineList forwardPage;
PipelineList backwardPage;
do { // For each limit (or page size) - forward scroll till the end
LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
forwardPage = listPipelines(null, null, limit, null, after, adminAuthHeaders());
printPipelines(forwardPage);
after = forwardPage.getPaging().getAfter();
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allPipelines.getData(), forwardPage, limit, indexInAllPipelines);
if (pageCount == 0) { // CASE 0 - First page is being returned. There is no before cursor
assertNull(before);
} else {
// Make sure scrolling back based on before cursor returns the correct result
backwardPage = listPipelines(null, null, limit, before, null, adminAuthHeaders());
assertEntityPagination(allPipelines.getData(), backwardPage, limit, (indexInAllPipelines - limit));
}
indexInAllPipelines += forwardPage.getData().size();
pageCount++;
} while (after != null);
// We have now reached the last page - test backward scroll till the beginning
pageCount = 0;
indexInAllPipelines = totalRecords - limit - forwardPage.getData().size();
do {
LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
forwardPage = listPipelines(null, null, limit, before, null, adminAuthHeaders());
printPipelines(forwardPage);
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allPipelines.getData(), forwardPage, limit, indexInAllPipelines);
pageCount++;
indexInAllPipelines -= forwardPage.getData().size();
} while (before != null);
}
}
private void printPipelines(PipelineList list) {
list.getData().forEach(Pipeline -> LOG.info("DB {}", Pipeline.getFullyQualifiedName()));
LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
@Test
public void put_PipelineUrlUpdate_200(TestInfo test) throws HttpResponseException, URISyntaxException {
CreatePipeline request = create(test).withService(new EntityReference().withId(AIRFLOW_REFERENCE.getId())
@ -512,23 +425,6 @@ public class PipelineResourceTest extends EntityResourceTest<Pipeline> {
return TestUtils.get(target, Pipeline.class, authHeaders);
}
public static PipelineList listPipelines(String fields, String serviceParam, Map<String, String> authHeaders)
throws HttpResponseException {
return listPipelines(fields, serviceParam, null, null, null, authHeaders);
}
public static PipelineList listPipelines(String fields, String serviceParam, Integer limitParam,
String before, String after, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = getResource("pipelines");
target = fields != null ? target.queryParam("fields", fields): target;
target = serviceParam != null ? target.queryParam("service", serviceParam): target;
target = limitParam != null ? target.queryParam("limit", limitParam): target;
target = before != null ? target.queryParam("before", before) : target;
target = after != null ? target.queryParam("after", after) : target;
return TestUtils.get(target, PipelineList.class, authHeaders);
}
private void deletePipeline(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
TestUtils.delete(getResource("pipelines/" + id), authHeaders);

112
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java
View File

@ -29,7 +29,6 @@ import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.jdbi3.TeamRepository.TeamEntityInterface;
import org.openmetadata.catalog.jdbi3.UserRepository.UserEntityInterface;
import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.resources.databases.TableResourceTest;
import org.openmetadata.catalog.resources.teams.TeamResource.TeamList;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
@ -67,17 +66,15 @@ import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityN
import static org.openmetadata.catalog.resources.teams.UserResourceTest.createUser;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.assertEntityPagination;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
import static org.openmetadata.catalog.util.TestUtils.validateEntityReference;
public class TeamResourceTest extends EntityResourceTest<Team> {
private static final Logger LOG = LoggerFactory.getLogger(TeamResourceTest.class);
final Profile PROFILE = new Profile().withImages(new ImageList().withImage(URI.create("http://image.com")));
public TeamResourceTest() {
super(Team.class, "teams", TeamResource.FIELDS, false);
super(Team.class, TeamList.class, "teams", TeamResource.FIELDS, false);
}
@Test
@ -199,97 +196,6 @@ public class TeamResourceTest extends EntityResourceTest<Team> {
assertResponse(exception, BAD_REQUEST, CatalogExceptionMessage.invalidField("invalidField"));
}
@Test
public void get_teamListWithInvalidLimit_4xx() {
// Limit must be >= 1 and <= 1000,000
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listTeams(null, -1, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listTeams(null, 0, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listTeams(null, 1000001, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be less than or equal to 1000000]");
}
@Test
public void get_teamListWithInvalidPaginationCursors_4xx() {
// Passing both before and after cursors is invalid
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listTeams(null, 1, "", "", adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "Only one of before or after query parameter allowed");
}
/**
* For cursor based pagination and implementation details:
* @see org.openmetadata.catalog.util.ResultList
*
* The tests and various CASES referenced are base on that.
*/
@Test
public void get_teamListWithPagination_200(TestInfo test) throws HttpResponseException {
// Create a large number of teams
int maxTeams = 40;
for (int i = 0; i < maxTeams; i++) {
createTeam(create(test, i), adminAuthHeaders());
}
// List all teams and use it for checking pagination
TeamList allTeams = listTeams(null, 1000000, null, null, adminAuthHeaders());
int totalRecords = allTeams.getData().size();
printTeams(allTeams);
// List tables with limit set from 1 to maxTables size
// Each time comapare the returned list with allTables list to make sure right results are returned
for (int limit = 1; limit < maxTeams; limit++) {
String after = null;
String before;
int pageCount = 0;
int indexInAllTables = 0;
TeamList forwardPage;
TeamList backwardPage;
do { // For each limit (or page size) - forward scroll till the end
LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
forwardPage = listTeams(null, limit, null, after, adminAuthHeaders());
after = forwardPage.getPaging().getAfter();
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allTeams.getData(), forwardPage, limit, indexInAllTables);
if (pageCount == 0) { // CASE 0 - First page is being returned. There is no before cursor
assertNull(before);
} else {
// Make sure scrolling back based on before cursor returns the correct result
backwardPage = listTeams(null, limit, before, null, adminAuthHeaders());
assertEntityPagination(allTeams.getData(), backwardPage, limit, (indexInAllTables - limit));
}
printTeams(forwardPage);
indexInAllTables += forwardPage.getData().size();
pageCount++;
} while (after != null);
// We have now reached the last page - test backward scroll till the beginning
pageCount = 0;
indexInAllTables = totalRecords - limit - forwardPage.getData().size();
do {
LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
forwardPage = listTeams(null, limit, before, null, adminAuthHeaders());
printTeams(forwardPage);
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allTeams.getData(), forwardPage, limit, indexInAllTables);
pageCount++;
indexInAllTables -= forwardPage.getData().size();
} while (before != null);
}
}
private void printTeams(TeamList list) {
list.getData().forEach(team -> LOG.info("Team {}", team.getName()));
LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
/**
* @see EntityResourceTest#put_addDeleteFollower_200
* for tests related getting team with entities owned by the team
@ -424,17 +330,6 @@ public class TeamResourceTest extends EntityResourceTest<Team> {
return TestUtils.get(target, Team.class, authHeaders);
}
public static TeamList listTeams(String fields, Integer limit, String before, String after,
Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("teams");
target = fields != null ? target.queryParam("fields", fields) : target;
target = limit != null ? target.queryParam("limit", limit) : target;
target = before != null ? target.queryParam("before", before) : target;
target = after != null ? target.queryParam("after", after) : target;
return TestUtils.get(target, TeamList.class, authHeaders);
}
public static Team getTeamByName(String name, String fields, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("teams/name/" + name);
@ -495,6 +390,7 @@ public class TeamResourceTest extends EntityResourceTest<Team> {
return TestUtils.patch(CatalogApplicationTest.getResource("teams/" + teamId), patch,
Team.class, authHeaders);
}
private Team patchTeam(String originalJson, Team updated, Map<String, String> authHeaders)
throws JsonProcessingException, HttpResponseException {
return patchTeam(updated.getId(), originalJson, updated, authHeaders);
@ -517,8 +413,8 @@ public class TeamResourceTest extends EntityResourceTest<Team> {
}
@Override
public Object createRequest(TestInfo test, String description, String displayName, EntityReference owner) {
return create(test).withDescription(description).withDisplayName(displayName);
public Object createRequest(TestInfo test, int index, String description, String displayName, EntityReference owner) {
return create(test, index).withDescription(description).withDisplayName(displayName);
}
@Override

109
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/UserResourceTest.java
View File

@ -76,7 +76,6 @@ import static org.openmetadata.catalog.resources.teams.TeamResourceTest.createTe
import static org.openmetadata.catalog.util.TestUtils.LONG_ENTITY_NAME;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.assertEntityPagination;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
@ -85,7 +84,7 @@ public class UserResourceTest extends EntityResourceTest<User> {
final Profile PROFILE = new Profile().withImages(new ImageList().withImage(URI.create("http://image.com")));
public UserResourceTest() {
super(User.class, "users", UserResource.FIELDS, false);
super(User.class, UserList.class, "users", UserResource.FIELDS, false);
}
@Test
@ -266,98 +265,6 @@ public class UserResourceTest extends EntityResourceTest<User> {
assertResponse(exception, BAD_REQUEST, CatalogExceptionMessage.invalidField("invalidField"));
}
@Test
public void get_userListWithInvalidLimit_4xx() {
// Limit must be >= 1 and <= 1000,000
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listUsers(null, -1, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listUsers(null, 0, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listUsers(null, 1000001, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be less than or equal to 1000000]");
}
@Test
public void get_userListWithInvalidPaginationCursors_4xx() {
// Passing both before and after cursors is invalid
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listUsers(null, 1, "", "", adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "Only one of before or after query parameter allowed");
}
/**
* For cursor based pagination and implementation details:
* @see org.openmetadata.catalog.util.ResultList
*
* The tests and various CASES referenced are base on that.
*/
@Test
public void get_userListWithPagination_200(TestInfo test) throws HttpResponseException {
// Create a large number of users
int maxUsers = 40;
for (int i = 0; i < maxUsers; i++) {
createUser(create(test, i), adminAuthHeaders());
}
// List all users and use it for checking pagination
UserList allUsers = listUsers(null, 1000000, null, null, adminAuthHeaders());
int totalRecords = allUsers.getData().size();
printUsers(allUsers);
// List tables with limit set from 1 to maxTables size
// Each time compare the returned list with allTables list to make sure right results are returned
for (int limit = 1; limit < 2; limit++) {
String after = null;
String before;
int pageCount = 0;
int indexInAllTables = 0;
UserList forwardPage;
UserList backwardPage;
do { // For each limit (or page size) - forward scroll till the end
LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
forwardPage = listUsers(null, limit, null, after, adminAuthHeaders());
after = forwardPage.getPaging().getAfter();
before = forwardPage.getPaging().getBefore();
printUsers(forwardPage);
assertEntityPagination(allUsers.getData(), forwardPage, limit, indexInAllTables);
if (pageCount == 0) { // CASE 0 - First page is being returned. There is no before cursor
assertNull(before);
} else {
// Make sure scrolling back based on before cursor returns the correct result
backwardPage = listUsers(null, limit, before, null, adminAuthHeaders());
assertEntityPagination(allUsers.getData(), backwardPage, limit, (indexInAllTables - limit));
}
indexInAllTables += forwardPage.getData().size();
pageCount++;
} while (after != null);
// We have now reached the last page - test backward scroll till the beginning
pageCount = 0;
indexInAllTables = totalRecords - limit - forwardPage.getData().size();
do {
LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
forwardPage = listUsers(null, limit, before, null, adminAuthHeaders());
printUsers(forwardPage);
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allUsers.getData(), forwardPage, limit, indexInAllTables);
pageCount++;
indexInAllTables -= forwardPage.getData().size();
} while (before != null);
}
}
private void printUsers(UserList list) {
list.getData().forEach(user -> LOG.info("User {}", user.getName()));
LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
/**
* @see EntityResourceTest#put_addDeleteFollower_200 test for tests related to GET user with owns field parameter
*
@ -575,16 +482,6 @@ public class UserResourceTest extends EntityResourceTest<User> {
TestUtils.delete(CatalogApplicationTest.getResource("users/" + id), headers);
}
public static UserList listUsers(String fields, Integer limit, String before, String after,
Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("users");
target = fields != null ? target.queryParam("fields", fields) : target;
target = limit != null ? target.queryParam("limit", limit) : target;
target = before != null ? target.queryParam("before", before) : target;
target = after != null ? target.queryParam("after", after) : target;
return TestUtils.get(target, UserList.class, authHeaders);
}
// TODO write following tests
// list users
// list users with various fields parameters
@ -595,8 +492,8 @@ public class UserResourceTest extends EntityResourceTest<User> {
}
@Override
public Object createRequest(TestInfo test, String description, String displayName, EntityReference owner) {
return create(test).withDescription(description).withDisplayName(displayName);
public Object createRequest(TestInfo test, int index, String description, String displayName, EntityReference owner) {
return create(test, index).withDescription(description).withDisplayName(displayName);
}
@Override

120
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/topics/TopicResourceTest.java
View File

@ -31,14 +31,14 @@ import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.TagLabel;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.ResultList;
import org.openmetadata.catalog.util.TestUtils;
import org.openmetadata.catalog.util.TestUtils.UpdateType;
import org.openmetadata.common.utils.JsonSchemaUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.json.JsonPatch;
import javax.ws.rs.client.WebTarget;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@ -56,15 +56,13 @@ import static org.openmetadata.catalog.util.TestUtils.LONG_ENTITY_NAME;
import static org.openmetadata.catalog.util.TestUtils.NON_EXISTENT_ENTITY;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.assertEntityPagination;
import static org.openmetadata.catalog.util.TestUtils.assertResponse;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
public class TopicResourceTest extends EntityResourceTest<Topic> {
private static final Logger LOG = LoggerFactory.getLogger(TopicResourceTest.class);
public TopicResourceTest() {
super(Topic.class, "topics", TopicResource.FIELDS, true);
super(Topic.class, TopicList.class, "topics", TopicResource.FIELDS, true);
}
@Test
@ -169,100 +167,14 @@ public class TopicResourceTest extends EntityResourceTest<Topic> {
createAndCheckEntity(create(test).withService(service), adminAuthHeaders());
// List topics by filtering on service name and ensure right topics are returned in the response
TopicList list = listTopics("service", service.getName(), adminAuthHeaders());
Map<String, String> queryParams = new HashMap<>(){{put("service", service.getName());}};
ResultList<Topic> list = listEntities(queryParams, adminAuthHeaders());
for (Topic topic : list.getData()) {
assertEquals(service.getName(), topic.getService().getName());
}
}
}
@Test
public void get_topicListWithInvalidLimitOffset_4xx() {
// Limit must be >= 1 and <= 1000,000
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listTopics(null, null, -1, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listTopics(null, null, 0, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be greater than or equal to 1]");
exception = assertThrows(HttpResponseException.class, ()
-> listTopics(null, null, 1000001, null, null, adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "[query param limit must be less than or equal to 1000000]");
}
@Test
public void get_topicListWithInvalidPaginationCursors_4xx() {
// Passing both before and after cursors is invalid
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> listTopics(null, null, 1, "", "", adminAuthHeaders()));
assertResponse(exception, BAD_REQUEST, "Only one of before or after query parameter allowed");
}
@Test
public void get_topicListWithValidLimitOffset_4xx(TestInfo test) throws HttpResponseException {
// Create a large number of topics
int maxTopics = 40;
for (int i = 0; i < maxTopics; i++) {
createTopic(create(test, i), adminAuthHeaders());
}
// List all topics
TopicList allTopics = listTopics(null, null, 1000000, null,
null, adminAuthHeaders());
int totalRecords = allTopics.getData().size();
printTopics(allTopics);
// List limit number topics at a time at various offsets and ensure right results are returned
for (int limit = 1; limit < maxTopics; limit++) {
String after = null;
String before;
int pageCount = 0;
int indexInAllTopics = 0;
TopicList forwardPage;
TopicList backwardPage;
do { // For each limit (or page size) - forward scroll till the end
LOG.info("Limit {} forward scrollCount {} afterCursor {}", limit, pageCount, after);
forwardPage = listTopics(null, null, limit, null, after, adminAuthHeaders());
printTopics(forwardPage);
after = forwardPage.getPaging().getAfter();
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allTopics.getData(), forwardPage, limit, indexInAllTopics);
if (pageCount == 0) { // CASE 0 - First page is being returned. There is no before cursor
assertNull(before);
} else {
// Make sure scrolling back based on before cursor returns the correct result
backwardPage = listTopics(null, null, limit, before, null, adminAuthHeaders());
assertEntityPagination(allTopics.getData(), backwardPage, limit, (indexInAllTopics - limit));
}
indexInAllTopics += forwardPage.getData().size();
pageCount++;
} while (after != null);
// We have now reached the last page - test backward scroll till the beginning
pageCount = 0;
indexInAllTopics = totalRecords - limit - forwardPage.getData().size();
do {
LOG.info("Limit {} backward scrollCount {} beforeCursor {}", limit, pageCount, before);
forwardPage = listTopics(null, null, limit, before, null, adminAuthHeaders());
printTopics(forwardPage);
before = forwardPage.getPaging().getBefore();
assertEntityPagination(allTopics.getData(), forwardPage, limit, indexInAllTopics);
pageCount++;
indexInAllTopics -= forwardPage.getData().size();
} while (before != null);
}
}
private void printTopics(TopicList list) {
list.getData().forEach(topic -> LOG.info("Topic {}", topic.getFullyQualifiedName()));
LOG.info("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
@Test
public void get_nonExistentTopic_404_notFound() {
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
@ -451,23 +363,6 @@ public class TopicResourceTest extends EntityResourceTest<Topic> {
return TestUtils.get(target, Topic.class, authHeaders);
}
public static TopicList listTopics(String fields, String serviceParam, Map<String, String> authHeaders)
throws HttpResponseException {
return listTopics(fields, serviceParam, null, null, null, authHeaders);
}
public static TopicList listTopics(String fields, String serviceParam, Integer limitParam,
String before, String after, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = getResource("topics");
target = fields != null ? target.queryParam("fields", fields) : target;
target = serviceParam != null ? target.queryParam("service", serviceParam) : target;
target = limitParam != null ? target.queryParam("limit", limitParam) : target;
target = before != null ? target.queryParam("before", before) : target;
target = after != null ? target.queryParam("after", after) : target;
return TestUtils.get(target, TopicList.class, authHeaders);
}
private void deleteTopic(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
TestUtils.delete(getResource("topics/" + id), authHeaders);
@ -494,8 +389,9 @@ public class TopicResourceTest extends EntityResourceTest<Topic> {
}
@Override
public CreateTopic createRequest(TestInfo test, String description, String displayName, EntityReference owner) {
return create(test).withDescription(description).withOwner(owner);
public CreateTopic createRequest(TestInfo test, int index, String description, String displayName,
EntityReference owner) {
return create(test, index).withDescription(description).withOwner(owner);
}
@Override