Fixes #1325 - Not getting cursor based paging for Messaging and Pipeline services

This commit is contained in:
sureshms 2021-11-23 21:39:44 -08:00
parent 80ae01c737
commit 41a281338a
12 changed files with 494 additions and 278 deletions

View File

@ -49,7 +49,6 @@ public class CatalogGenericExceptionMapper implements ExceptionMapper<Throwable>
@Override
public Response toResponse(Throwable ex) {
LOG.info("Exception ", ex);
if (ex instanceof ProcessingException || ex instanceof IllegalArgumentException) {
final Response response = BadRequestException.of().getResponse();
return Response.fromResponse(response)

View File

@ -53,16 +53,6 @@ public class DatabaseServiceRepository extends EntityRepository<DatabaseService>
this.dao = dao;
}
public DatabaseService update(UriInfo uriInfo, UUID id, String description, JdbcInfo jdbc, Schedule ingestionSchedule)
throws IOException {
EntityUtil.validateIngestionSchedule(ingestionSchedule);
DatabaseService dbService = dao.dbServiceDAO().findEntityById(id);
// Update fields
dbService.withDescription(description).withJdbc((jdbc)).withIngestionSchedule(ingestionSchedule);
dao.dbServiceDAO().update(id, JsonUtils.pojoToJson(dbService));
return withHref(uriInfo, dbService);
}
@Transaction
public void delete(UUID id) {
if (dao.dbServiceDAO().delete(id) <= 0) {

View File

@ -51,19 +51,6 @@ public class PipelineServiceRepository extends EntityRepository<PipelineService>
this.dao = dao;
}
@Transaction
public PipelineService update(UriInfo uriInfo, UUID id, String description, URI url,
Schedule ingestionSchedule)
throws IOException {
EntityUtil.validateIngestionSchedule(ingestionSchedule);
PipelineService pipelineService = dao.pipelineServiceDAO().findEntityById(id);
// Update fields
pipelineService.withDescription(description).withIngestionSchedule(ingestionSchedule)
.withPipelineUrl(url);
dao.pipelineServiceDAO().update(id, JsonUtils.pojoToJson(pipelineService));
return withHref(uriInfo, pipelineService);
}
@Transaction
public void delete(UUID id) {
if (dao.pipelineServiceDAO().delete(id) <= 0) {

View File

@ -47,15 +47,6 @@ public class StorageServiceRepository extends EntityRepository<StorageService> {
this.dao = dao;
}
public StorageService update(UriInfo uriInfo, UUID id, String description)
throws IOException {
StorageService storageService = dao.storageServiceDAO().findEntityById(id);
// Update fields
storageService.withDescription(description);
dao.storageServiceDAO().update(id, JsonUtils.pojoToJson(storageService));
return withHref(uriInfo, storageService);
}
@Transaction
public void delete(UUID id) {
if (dao.storageServiceDAO().delete(id) <= 0) {

View File

@ -30,6 +30,7 @@ import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository;
import org.openmetadata.catalog.resources.Collection;
import org.openmetadata.catalog.security.CatalogAuthorizer;
import org.openmetadata.catalog.security.SecurityUtil;
import org.openmetadata.catalog.type.EntityHistory;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.util.RestUtil.PutResponse;
import org.openmetadata.catalog.util.ResultList;
@ -53,6 +54,7 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Date;
@ -78,8 +80,12 @@ public class DatabaseServiceResource {
}
public static class DatabaseServiceList extends ResultList<DatabaseService> {
public DatabaseServiceList(List<DatabaseService> data) {
super(data);
@SuppressWarnings("unused") /* Required for tests */
public DatabaseServiceList() {}
public DatabaseServiceList(List<DatabaseService> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(data, beforeCursor, afterCursor, total);
}
}
@ -105,7 +111,6 @@ public class DatabaseServiceResource {
throws IOException, GeneralSecurityException, ParseException {
RestUtil.validateCursors(before, after);
ResultList<DatabaseService> list;
if(before != null) {
return dao.listBefore(uriInfo, null, null, limitParam, before);
}
@ -144,6 +149,44 @@ public class DatabaseServiceResource {
return dao.getByName(uriInfo, name, null);
}
@GET
@Path("/{id}/versions")
@Operation(summary = "List database service versions", tags = "services",
description = "Get a list of all the versions of a database service identified by `id`",
responses = {@ApiResponse(responseCode = "200", description = "List of database service versions",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "database service Id", schema = @Schema(type = "string"))
@PathParam("id") String id)
throws IOException, ParseException, GeneralSecurityException {
return dao.listVersions(id);
}
@GET
@Path("/{id}/versions/{version}")
@Operation(summary = "Get a version of the database service", tags = "services",
description = "Get a version of the database service by given `id`",
responses = {
@ApiResponse(responseCode = "200", description = "database service",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = DatabaseService.class))),
@ApiResponse(responseCode = "404", description = "Database service for instance {id} and version " +
"{version} is not found")
})
public DatabaseService getVersion(@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "database service Id", schema = @Schema(type = "string"))
@PathParam("id") String id,
@Parameter(description = "database service version number in the form `major`" +
".`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version") String version) throws IOException, ParseException {
return dao.getVersion(id, version);
}
@POST
@Operation(summary = "Create database service", tags = "services",
description = "Create a new database service.",

View File

@ -178,16 +178,17 @@ public class MessagingServiceResource {
@ApiResponse(responseCode = "200", description = "messaging service",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = MessagingService.class))),
@ApiResponse(responseCode = "404", description = "Messaging s3rvice for instance {id} and version " +
@ApiResponse(responseCode = "404", description = "Messaging service for instance {id} and version " +
"{version} is not found")
})
public MessagingService getVersion(@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "messaging service Id", schema = @Schema(type = "string"))
@PathParam("id") String id,
@Parameter(description = "messaging service version number in the form `major`.`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version") String version) throws IOException, ParseException {
@Context SecurityContext securityContext,
@Parameter(description = "messaging service Id", schema = @Schema(type = "string"))
@PathParam("id") String id,
@Parameter(description = "messaging service version number in the form `major`" +
".`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version") String version) throws IOException, ParseException {
return dao.getVersion(id, version);
}

View File

@ -28,9 +28,9 @@ import org.openmetadata.catalog.entity.services.PipelineService;
import org.openmetadata.catalog.jdbi3.CollectionDAO;
import org.openmetadata.catalog.jdbi3.PipelineServiceRepository;
import org.openmetadata.catalog.resources.Collection;
import org.openmetadata.catalog.resources.databases.TableResource.TableList;
import org.openmetadata.catalog.security.CatalogAuthorizer;
import org.openmetadata.catalog.security.SecurityUtil;
import org.openmetadata.catalog.type.EntityHistory;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.util.RestUtil.PutResponse;
@ -55,6 +55,7 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Date;
@ -84,8 +85,12 @@ public class PipelineServiceResource {
}
public static class PipelineServiceList extends ResultList<PipelineService> {
public PipelineServiceList(List<PipelineService> data) {
super(data);
@SuppressWarnings("unused") /* Required for tests */
public PipelineServiceList() {}
public PipelineServiceList(List<PipelineService> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(data, beforeCursor, afterCursor, total);
}
}
@ -95,7 +100,7 @@ public class PipelineServiceResource {
"entries in the list using `limit` and `before` or `after` query params.",
responses = {@ApiResponse(responseCode = "200", description = "List of pipeline services",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = TableList.class)))
schema = @Schema(implementation = PipelineServiceList.class)))
})
public ResultList<PipelineService> list(@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@ -112,6 +117,8 @@ public class PipelineServiceResource {
schema = @Schema(type = "string"))
@QueryParam("after") String after) throws IOException,
GeneralSecurityException, ParseException {
RestUtil.validateCursors(before, after);
if (before != null) { // Reverse paging
return dao.listBefore(uriInfo, null, null, limitParam, before);
}
@ -151,6 +158,44 @@ public class PipelineServiceResource {
return dao.getByName(uriInfo, name, null);
}
@GET
@Path("/{id}/versions")
@Operation(summary = "List pipeline service versions", tags = "services",
description = "Get a list of all the versions of a pipeline service identified by `id`",
responses = {@ApiResponse(responseCode = "200", description = "List of pipeline service versions",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "pipeline service Id", schema = @Schema(type = "string"))
@PathParam("id") String id)
throws IOException, ParseException, GeneralSecurityException {
return dao.listVersions(id);
}
@GET
@Path("/{id}/versions/{version}")
@Operation(summary = "Get a version of the pipeline service", tags = "services",
description = "Get a version of the pipeline service by given `id`",
responses = {
@ApiResponse(responseCode = "200", description = "pipeline service",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = PipelineService.class))),
@ApiResponse(responseCode = "404", description = "Pipeline service for instance {id} and version " +
"{version} is not found")
})
public PipelineService getVersion(@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "pipeline service Id", schema = @Schema(type = "string"))
@PathParam("id") String id,
@Parameter(description = "pipeline service version number in the form `major`" +
".`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version") String version) throws IOException, ParseException {
return dao.getVersion(id, version);
}
@POST
@Operation(summary = "Create a pipeline service", tags = "services",
description = "Create a new pipeline service.",

View File

@ -28,9 +28,10 @@ import org.openmetadata.catalog.entity.services.StorageService;
import org.openmetadata.catalog.jdbi3.CollectionDAO;
import org.openmetadata.catalog.jdbi3.StorageServiceRepository;
import org.openmetadata.catalog.resources.Collection;
import org.openmetadata.catalog.resources.databases.TableResource.TableList;
import org.openmetadata.catalog.security.CatalogAuthorizer;
import org.openmetadata.catalog.security.SecurityUtil;
import org.openmetadata.catalog.type.EntityHistory;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.util.RestUtil.PutResponse;
import org.openmetadata.catalog.util.ResultList;
@ -53,6 +54,7 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.GeneralSecurityException;
import java.text.ParseException;
import java.util.Date;
@ -78,9 +80,13 @@ public class StorageServiceResource {
this.authorizer = authorizer;
}
static class StorageServiceList extends ResultList<StorageService> {
StorageServiceList(List<StorageService> data) {
super(data);
public static class StorageServiceList extends ResultList<StorageService> {
@SuppressWarnings("unused") /* Required for tests */
public StorageServiceList() {}
public StorageServiceList(List<StorageService> data, String beforeCursor, String afterCursor, int total)
throws GeneralSecurityException, UnsupportedEncodingException {
super(data, beforeCursor, afterCursor, total);
}
}
@ -90,7 +96,7 @@ public class StorageServiceResource {
"entries in the list using `limit` and `before` or `after` query params.",
responses = {@ApiResponse(responseCode = "200", description = "List of storage services",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = TableList.class)))
schema = @Schema(implementation = StorageServiceList.class)))
})
public ResultList<StorageService> list(@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@ -107,6 +113,7 @@ public class StorageServiceResource {
schema = @Schema(type = "string"))
@QueryParam("after") String after) throws IOException,
GeneralSecurityException, ParseException {
RestUtil.validateCursors(before, after);
if (before != null) { // Reverse paging
return dao.listBefore(uriInfo, null, null, limitParam, before);
}
@ -146,6 +153,44 @@ public class StorageServiceResource {
return dao.getByName(uriInfo, name, null);
}
@GET
@Path("/{id}/versions")
@Operation(summary = "List storage service versions", tags = "services",
description = "Get a list of all the versions of a storage service identified by `id`",
responses = {@ApiResponse(responseCode = "200", description = "List of storage service versions",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = EntityHistory.class)))
})
public EntityHistory listVersions(@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "storage service Id", schema = @Schema(type = "string"))
@PathParam("id") String id)
throws IOException, ParseException, GeneralSecurityException {
return dao.listVersions(id);
}
@GET
@Path("/{id}/versions/{version}")
@Operation(summary = "Get a version of the storage service", tags = "services",
description = "Get a version of the storage service by given `id`",
responses = {
@ApiResponse(responseCode = "200", description = "storage service",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = StorageService.class))),
@ApiResponse(responseCode = "404", description = "Storage service for instance {id} and version " +
"{version} is not found")
})
public StorageService getVersion(@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(description = "storage service Id", schema = @Schema(type = "string"))
@PathParam("id") String id,
@Parameter(description = "storage service version number in the form `major`" +
".`minor`",
schema = @Schema(type = "string", example = "0.1 or 1.1"))
@PathParam("version") String version) throws IOException, ParseException {
return dao.getVersion(id, version);
}
@POST
@Operation(summary = "Create storage service", tags = "services",
description = "Create a new storage service.",

View File

@ -622,6 +622,8 @@ public abstract class EntityResourceTest<T> extends CatalogApplicationTest {
assertEquals(0.1, entityInterface.getVersion()); // First version of the entity
validateCreatedEntity(getEntity, create, authHeaders);
// TODO GET the entity by name
// Validate that change event was created
validateChangeEvents(entityInterface, entityInterface.getUpdatedAt(), EventType.ENTITY_CREATED,
null, authHeaders);

View File

@ -25,12 +25,19 @@ import org.openmetadata.catalog.api.services.CreateDatabaseService;
import org.openmetadata.catalog.api.services.CreateDatabaseService.DatabaseServiceType;
import org.openmetadata.catalog.entity.services.DatabaseService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository.DatabaseServiceEntityInterface;
import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.resources.services.database.DatabaseServiceResource.DatabaseServiceList;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.JdbcInfo;
import org.openmetadata.catalog.type.Schedule;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.TestUtils;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response.Status;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Date;
import java.util.Map;
import java.util.UUID;
@ -46,7 +53,13 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
public class DatabaseServiceResourceTest extends CatalogApplicationTest {
public class DatabaseServiceResourceTest extends EntityResourceTest<DatabaseService> {
public DatabaseServiceResourceTest() {
super(Entity.DATABASE_SERVICE, DatabaseService.class, DatabaseServiceList.class, "services/databaseServices",
"", false, false, false);
this.supportsPatch = false;
}
@Test
public void post_databaseServiceWithLongName_400_badRequest(TestInfo test) {
// Create database with mandatory name field empty
@ -75,12 +88,12 @@ public class DatabaseServiceResourceTest extends CatalogApplicationTest {
}
@Test
public void post_validDatabaseService_as_admin_200_ok(TestInfo test) throws HttpResponseException {
public void post_validDatabaseService_as_admin_200_ok(TestInfo test) throws IOException {
// Create database service with different optional fields
Map<String, String> authHeaders = adminAuthHeaders();
createAndCheckService(create(test, 1).withDescription(null), authHeaders);
createAndCheckService(create(test, 2).withDescription("description"), authHeaders);
createAndCheckService(create(test, 3).withIngestionSchedule(null), authHeaders);
createAndCheckEntity(create(test, 1).withDescription(null), authHeaders);
createAndCheckEntity(create(test, 2).withDescription("description"), authHeaders);
createAndCheckEntity(create(test, 3).withIngestionSchedule(null), authHeaders);
}
@Test
@ -89,7 +102,7 @@ public class DatabaseServiceResourceTest extends CatalogApplicationTest {
Map<String, String> authHeaders = authHeaders("test@open-metadata.org");
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
createAndCheckService(create(test, 1).withDescription(null), authHeaders));
createAndCheckEntity(create(test, 1).withDescription(null), authHeaders));
TestUtils.assertResponse(exception, FORBIDDEN,
"Principal: CatalogPrincipal{name='test'} is not admin");
}
@ -136,16 +149,16 @@ public class DatabaseServiceResourceTest extends CatalogApplicationTest {
}
@Test
public void post_validIngestionSchedules_as_admin_200(TestInfo test) throws HttpResponseException {
public void post_validIngestionSchedules_as_admin_200(TestInfo test) throws IOException {
Schedule schedule = new Schedule().withStartDate(new Date());
schedule.withRepeatFrequency("PT60M"); // Repeat every 60M should be valid
createAndCheckService(create(test, 1).withIngestionSchedule(schedule), adminAuthHeaders());
createAndCheckEntity(create(test, 1).withIngestionSchedule(schedule), adminAuthHeaders());
schedule.withRepeatFrequency("PT1H49M");
createAndCheckService(create(test, 2).withIngestionSchedule(schedule), adminAuthHeaders());
createAndCheckEntity(create(test, 2).withIngestionSchedule(schedule), adminAuthHeaders());
schedule.withRepeatFrequency("P1DT1H49M");
createAndCheckService(create(test, 3).withIngestionSchedule(schedule), adminAuthHeaders());
createAndCheckEntity(create(test, 3).withIngestionSchedule(schedule), adminAuthHeaders());
}
@Test
@ -166,8 +179,8 @@ public class DatabaseServiceResourceTest extends CatalogApplicationTest {
}
@Test
public void put_updateDatabaseService_as_admin_2xx(TestInfo test) throws HttpResponseException {
createAndCheckService(create(test).withDescription(null).withIngestionSchedule(null), adminAuthHeaders());
public void put_updateDatabaseService_as_admin_2xx(TestInfo test) throws IOException {
createAndCheckEntity(create(test).withDescription(null).withIngestionSchedule(null), adminAuthHeaders());
// Update database description and ingestion service that are null
CreateDatabaseService update = create(test).withDescription("description1");
updateAndCheckService(update, OK, adminAuthHeaders());
@ -182,9 +195,9 @@ public class DatabaseServiceResourceTest extends CatalogApplicationTest {
}
@Test
public void put_update_as_non_admin_401(TestInfo test) throws HttpResponseException {
public void put_update_as_non_admin_401(TestInfo test) throws IOException {
Map<String, String> authHeaders = adminAuthHeaders();
createAndCheckService(create(test).withDescription(null).withIngestionSchedule(null), authHeaders);
createAndCheckEntity(create(test).withDescription(null).withIngestionSchedule(null), authHeaders);
// Update as non admin should be forbidden
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
@ -209,26 +222,6 @@ public class DatabaseServiceResourceTest extends CatalogApplicationTest {
"invalidName"));
}
public static DatabaseService createAndCheckService(CreateDatabaseService create,
Map<String, String> authHeaders) throws HttpResponseException {
String updatedBy = TestUtils.getPrincipal(authHeaders);
DatabaseService service = createService(create, authHeaders);
assertEquals(0.1, service.getVersion());
validateService(service, create.getName(), create.getDescription(), create.getJdbc(),
create.getIngestionSchedule(), updatedBy);
// GET the newly created service and validate
DatabaseService getService = getService(service.getId(), authHeaders);
validateService(getService, create.getName(), create.getDescription(), create.getJdbc(),
create.getIngestionSchedule(), updatedBy);
// GET the newly created service by name and validate
getService = getServiceByName(service.getName(), null, authHeaders);
validateService(getService, create.getName(), create.getDescription(), create.getJdbc(),
create.getIngestionSchedule(), updatedBy);
return service;
}
public static DatabaseService createService(CreateDatabaseService create,
Map<String, String> authHeaders) throws HttpResponseException {
return TestUtils.post(CatalogApplicationTest.getResource("services/databaseServices"),
@ -352,4 +345,38 @@ public class DatabaseServiceResourceTest extends CatalogApplicationTest {
return TestUtils.put(CatalogApplicationTest.getResource("services/databaseServices"), updated,
DatabaseService.class, status, authHeaders);
}
@Override
public Object createRequest(TestInfo test, int index, String description, String displayName, EntityReference owner)
throws URISyntaxException {
return create(test, index).withDescription(description);
}
@Override
public void validateCreatedEntity(DatabaseService createdEntity, Object request, Map<String, String> authHeaders)
throws HttpResponseException {
}
@Override
public void validateUpdatedEntity(DatabaseService updatedEntity, Object request, Map<String, String> authHeaders)
throws HttpResponseException {
}
@Override
public void compareEntities(DatabaseService expected, DatabaseService updated, Map<String, String> authHeaders)
throws HttpResponseException {
}
@Override
public EntityInterface<DatabaseService> getEntityInterface(DatabaseService entity) {
return new DatabaseServiceEntityInterface(entity);
}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
}
}

View File

@ -25,12 +25,17 @@ import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.services.CreatePipelineService;
import org.openmetadata.catalog.entity.services.PipelineService;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.jdbi3.PipelineServiceRepository.PipelineServiceEntityInterface;
import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.resources.services.pipeline.PipelineServiceResource.PipelineServiceList;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.Schedule;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.TestUtils;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response.Status;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Date;
@ -49,15 +54,55 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
public class PipelineServiceResourceTest extends CatalogApplicationTest {
public class PipelineServiceResourceTest extends EntityResourceTest<PipelineService> {
public static URI PIPELINE_SERVICE_URL;
public PipelineServiceResourceTest() {
super(Entity.PIPELINE_SERVICE, PipelineService.class, PipelineServiceList.class,
"services/pipelineServices", "", false, false, false);
this.supportsPatch = false;
}
@BeforeAll
public static void setup(TestInfo test) throws URISyntaxException {
public static void setup() throws URISyntaxException {
PIPELINE_SERVICE_URL = new URI("http://localhost:8080");
}
@Override
public Object createRequest(TestInfo test, int index, String description, String displayName, EntityReference owner)
throws URISyntaxException {
return create(test, index).withDescription(description).withIngestionSchedule(null);
}
@Override
public void validateCreatedEntity(PipelineService createdEntity, Object request, Map<String, String> authHeaders)
throws HttpResponseException {
}
@Override
public void validateUpdatedEntity(PipelineService updatedEntity, Object request, Map<String, String> authHeaders)
throws HttpResponseException {
}
@Override
public void compareEntities(PipelineService expected, PipelineService updated, Map<String, String> authHeaders)
throws HttpResponseException {
}
@Override
public EntityInterface<PipelineService> getEntityInterface(PipelineService entity) {
return new PipelineServiceEntityInterface(entity);
}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
}
@Test
public void post_serviceWithLongName_400_badRequest(TestInfo test) {
// Create pipeline with mandatory name field empty

View File

@ -7,14 +7,19 @@ import org.openmetadata.catalog.CatalogApplicationTest;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.api.services.CreateStorageService;
import org.openmetadata.catalog.entity.services.StorageService;
import org.openmetadata.catalog.type.StorageServiceType;
import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.util.RestUtil;
import org.openmetadata.catalog.jdbi3.StorageServiceRepository.StorageServiceEntityInterface;
import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.resources.services.storage.StorageServiceResource.StorageServiceList;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.StorageServiceType;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.TestUtils;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import java.util.Date;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Map;
import java.util.UUID;
@ -29,217 +34,253 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
import static org.openmetadata.catalog.util.TestUtils.authHeaders;
public class StorageServiceResourceTest extends CatalogApplicationTest {
@Test
public void post_ServiceWithLongName_400_badRequest(TestInfo test) {
// Create storage with mandatory name field empty
CreateStorageService create = create(test).withName(TestUtils.LONG_ENTITY_NAME);
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
createService(create, adminAuthHeaders()));
TestUtils.assertResponse(exception, BAD_REQUEST, "[name size must be between 1 and 64]");
}
public class StorageServiceResourceTest extends EntityResourceTest<StorageService> {
public StorageServiceResourceTest() {
super(Entity.STORAGE_SERVICE, StorageService.class, StorageServiceList.class,
"services/storageServices", "", false, false, false);
this.supportsPatch = false;
}
@Test
public void post_ServiceWithoutName_400_badRequest(TestInfo test) {
// Create storage with mandatory name field empty
CreateStorageService create = create(test).withName("");
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
createService(create, adminAuthHeaders()));
TestUtils.assertResponse(exception, BAD_REQUEST, "[name size must be between 1 and 64]");
}
@Test
public void post_ServiceAlreadyExists_409(TestInfo test) throws HttpResponseException {
CreateStorageService create = create(test);
createService(create, adminAuthHeaders());
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
createService(create, adminAuthHeaders()));
TestUtils.assertResponse(exception, CONFLICT, CatalogExceptionMessage.ENTITY_ALREADY_EXISTS);
}
@Test
public void post_ServiceWithLongName_400_badRequest(TestInfo test) {
// Create storage with mandatory name field empty
CreateStorageService create = create(test).withName(TestUtils.LONG_ENTITY_NAME);
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
createService(create, adminAuthHeaders()));
TestUtils.assertResponse(exception, BAD_REQUEST, "[name size must be between 1 and 64]");
}
@Test
public void post_validService_as_admin_200_ok(TestInfo test) throws HttpResponseException {
// Create storage service with different optional fields
Map<String, String> authHeaders = adminAuthHeaders();
createAndCheckService(create(test, 1).withDescription(null), authHeaders);
createAndCheckService(create(test, 2).withDescription("description"), authHeaders);
}
@Test
public void post_validService_as_non_admin_401(TestInfo test) {
// Create storage service with different optional fields
Map<String, String> authHeaders = authHeaders("test@open-metadata.org");
@Test
public void post_ServiceWithoutName_400_badRequest(TestInfo test) {
// Create storage with mandatory name field empty
CreateStorageService create = create(test).withName("");
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
createService(create, adminAuthHeaders()));
TestUtils.assertResponse(exception, BAD_REQUEST, "[name size must be between 1 and 64]");
}
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
createAndCheckService(create(test, 1).withDescription(null), authHeaders));
TestUtils.assertResponse(exception, FORBIDDEN,
"Principal: CatalogPrincipal{name='test'} is not admin");
}
@Test
public void post_ServiceAlreadyExists_409(TestInfo test) throws HttpResponseException {
CreateStorageService create = create(test);
createService(create, adminAuthHeaders());
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
createService(create, adminAuthHeaders()));
TestUtils.assertResponse(exception, CONFLICT, CatalogExceptionMessage.ENTITY_ALREADY_EXISTS);
}
@Test
public void put_updateStorageService_as_admin_2xx(TestInfo test) throws HttpResponseException {
StorageService dbService = createAndCheckService(create(test).withDescription(null), adminAuthHeaders());
String id = dbService.getId().toString();
@Test
public void post_validService_as_admin_200_ok(TestInfo test) throws HttpResponseException {
// Create storage service with different optional fields
Map<String, String> authHeaders = adminAuthHeaders();
createAndCheckService(create(test, 1).withDescription(null), authHeaders);
createAndCheckService(create(test, 2).withDescription("description"), authHeaders);
}
// Update storage description and ingestion service that are null
CreateStorageService update = create(test).withDescription("description1");
updateAndCheckService(update, OK, adminAuthHeaders());
@Test
public void post_validService_as_non_admin_401(TestInfo test) {
// Create storage service with different optional fields
Map<String, String> authHeaders = authHeaders("test@open-metadata.org");
// Update description and ingestion schedule again
update.withDescription("description1");
updateAndCheckService(update, OK, adminAuthHeaders());
}
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
createAndCheckService(create(test, 1).withDescription(null), authHeaders));
TestUtils.assertResponse(exception, FORBIDDEN,
"Principal: CatalogPrincipal{name='test'} is not admin");
}
@Test
public void put_update_as_non_admin_401(TestInfo test) throws HttpResponseException {
Map<String, String> authHeaders = adminAuthHeaders();
StorageService dbService = createAndCheckService(create(test).withDescription(null), authHeaders);
@Test
public void put_updateStorageService_as_admin_2xx(TestInfo test) throws HttpResponseException {
StorageService dbService = createAndCheckService(create(test).withDescription(null), adminAuthHeaders());
String id = dbService.getId().toString();
// Update storage description and ingestion service that are null
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
updateAndCheckService(create(test), OK, authHeaders("test@open-metadata.org")));
TestUtils.assertResponse(exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} " +
"is not admin");
}
// Update storage description and ingestion service that are null
CreateStorageService update = create(test).withDescription("description1");
updateAndCheckService(update, OK, adminAuthHeaders());
@Test
public void get_nonExistentStorageService_404_notFound() {
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
getService(TestUtils.NON_EXISTENT_ENTITY, adminAuthHeaders()));
TestUtils.assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.STORAGE_SERVICE,
TestUtils.NON_EXISTENT_ENTITY));
}
// Update description and ingestion schedule again
update.withDescription("description1");
updateAndCheckService(update, OK, adminAuthHeaders());
}
@Test
public void get_nonExistentStorageServiceByName_404_notFound() {
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> getServiceByName("invalidName", null, adminAuthHeaders()));
TestUtils.assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.STORAGE_SERVICE,
"invalidName"));
}
@Test
public void put_update_as_non_admin_401(TestInfo test) throws HttpResponseException {
Map<String, String> authHeaders = adminAuthHeaders();
StorageService dbService = createAndCheckService(create(test).withDescription(null), authHeaders);
@Test
public void delete_ExistentService_as_admin_200(TestInfo test) throws HttpResponseException {
Map<String, String> authHeaders = adminAuthHeaders();
StorageService storageService = createService(create(test), authHeaders);
deleteService(storageService.getId(), storageService.getName(), authHeaders);
}
// Update storage description and ingestion service that are null
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
updateAndCheckService(create(test), OK, authHeaders("test@open-metadata.org")));
TestUtils.assertResponse(exception, FORBIDDEN, "Principal: CatalogPrincipal{name='test'} " +
"is not admin");
}
@Test
public void delete_as_user_401(TestInfo test) throws HttpResponseException {
Map<String, String> authHeaders = adminAuthHeaders();
StorageService storageService = createService(create(test), authHeaders);
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
deleteService(storageService.getId(), storageService.getName(),
authHeaders("test@open-metadata.org")));
TestUtils.assertResponse(exception, FORBIDDEN,
"Principal: CatalogPrincipal{name='test'} is not admin");
}
@Test
public void get_nonExistentStorageService_404_notFound() {
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
getService(TestUtils.NON_EXISTENT_ENTITY, adminAuthHeaders()));
TestUtils.assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.STORAGE_SERVICE,
TestUtils.NON_EXISTENT_ENTITY));
}
@Test
public void delete_notExistentStorageService() {
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
getService(TestUtils.NON_EXISTENT_ENTITY, adminAuthHeaders()));
TestUtils.assertResponse(exception, NOT_FOUND,
CatalogExceptionMessage.entityNotFound(Entity.STORAGE_SERVICE, TestUtils.NON_EXISTENT_ENTITY));
}
public static CreateStorageService create(TestInfo test) {
return new CreateStorageService().withName(getName(test))
.withServiceType(StorageServiceType.S3);
}
@Test
public void get_nonExistentStorageServiceByName_404_notFound() {
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> getServiceByName("invalidName", null, adminAuthHeaders()));
TestUtils.assertResponse(exception, NOT_FOUND, CatalogExceptionMessage.entityNotFound(Entity.STORAGE_SERVICE,
"invalidName"));
}
private static CreateStorageService create(TestInfo test, int index) {
return new CreateStorageService().withName(getName(test, index))
.withServiceType(StorageServiceType.S3);
}
@Test
public void delete_ExistentService_as_admin_200(TestInfo test) throws HttpResponseException {
Map<String, String> authHeaders = adminAuthHeaders();
StorageService storageService = createService(create(test), authHeaders);
deleteService(storageService.getId(), storageService.getName(), authHeaders);
}
public static String getName(TestInfo test) {
return String.format("storage_service_%s", test.getDisplayName());
}
@Test
public void delete_as_user_401(TestInfo test) throws HttpResponseException {
Map<String, String> authHeaders = adminAuthHeaders();
StorageService storageService = createService(create(test), authHeaders);
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
deleteService(storageService.getId(), storageService.getName(),
authHeaders("test@open-metadata.org")));
TestUtils.assertResponse(exception, FORBIDDEN,
"Principal: CatalogPrincipal{name='test'} is not admin");
}
public static String getName(TestInfo test, int index) {
return String.format("storage_service_%d_%s", index, test.getDisplayName());
}
public static StorageService createService(CreateStorageService create,
Map<String, String> authHeaders) throws HttpResponseException {
return TestUtils.post(CatalogApplicationTest.getResource("services/storageServices"),
create, StorageService.class, authHeaders);
}
@Test
public void delete_notExistentStorageService() {
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
getService(TestUtils.NON_EXISTENT_ENTITY, adminAuthHeaders()));
TestUtils.assertResponse(exception, NOT_FOUND,
CatalogExceptionMessage.entityNotFound(Entity.STORAGE_SERVICE, TestUtils.NON_EXISTENT_ENTITY));
}
public static StorageService createAndCheckService(CreateStorageService create,
Map<String, String> authHeaders) throws HttpResponseException {
StorageService service = createService(create, authHeaders);
validateService(service, create.getName(), create.getDescription());
public static CreateStorageService create(TestInfo test) {
return new CreateStorageService().withName(getName(test)).withServiceType(StorageServiceType.S3);
}
// GET the newly created service and validate
StorageService getService = getService(service.getId(), authHeaders);
validateService(getService, create.getName(), create.getDescription());
private static CreateStorageService create(TestInfo test, int index) {
return new CreateStorageService().withName(getName(test, index))
.withServiceType(StorageServiceType.S3);
}
// GET the newly created service by name and validate
getService = getServiceByName(service.getName(), null, authHeaders);
validateService(getService, create.getName(), create.getDescription());
return service;
}
public static String getName(TestInfo test) {
return String.format("storageSvc_%s", test.getDisplayName());
}
private static void validateService(StorageService service, String expectedName, String expectedDescription) {
assertNotNull(service.getId());
assertNotNull(service.getHref());
assertEquals(expectedName, service.getName());
assertEquals(expectedDescription, service.getDescription());
}
public static String getName(TestInfo test, int index) {
return String.format("storageSvc_%d_%s", index, test.getDisplayName());
}
public static StorageService getService(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
return getService(id, null, authHeaders);
}
public static StorageService getService(UUID id, String fields, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("services/storageServices/" + id);
target = fields != null ? target.queryParam("fields", fields) : target;
return TestUtils.get(target, StorageService.class, authHeaders);
}
public static StorageService getServiceByName(String name, String fields, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("services/storageServices/name/" + name);
target = fields != null ? target.queryParam("fields", fields) : target;
return TestUtils.get(target, StorageService.class, authHeaders);
}
public static StorageService updateStorageService(CreateStorageService updated,
Response.Status status, Map<String, String> authHeaders)
throws HttpResponseException {
return TestUtils.put(CatalogApplicationTest.getResource("services/storageServices"), updated,
StorageService.class, status, authHeaders);
}
public static void updateAndCheckService(CreateStorageService update, Response.Status status,
public static StorageService createService(CreateStorageService create,
Map<String, String> authHeaders) throws HttpResponseException {
StorageService service = updateStorageService(update, status, authHeaders);
validateService(service, service.getName(), update.getDescription());
return TestUtils.post(CatalogApplicationTest.getResource("services/storageServices"),
create, StorageService.class, authHeaders);
}
// GET the newly updated storage and validate
StorageService getService = getService(service.getId(), authHeaders);
validateService(getService, service.getName(), update.getDescription());
public static StorageService createAndCheckService(CreateStorageService create,
Map<String, String> authHeaders) throws HttpResponseException {
StorageService service = createService(create, authHeaders);
validateService(service, create.getName(), create.getDescription());
// GET the newly updated storage by name and validate
getService = getServiceByName(service.getName(), null, authHeaders);
validateService(getService, service.getName(), update.getDescription());
}
// GET the newly created service and validate
StorageService getService = getService(service.getId(), authHeaders);
validateService(getService, create.getName(), create.getDescription());
private void deleteService(UUID id, String name, Map<String, String> authHeaders) throws HttpResponseException {
TestUtils.delete(CatalogApplicationTest.getResource("services/storageServices/" + id), authHeaders);
// GET the newly created service by name and validate
getService = getServiceByName(service.getName(), null, authHeaders);
validateService(getService, create.getName(), create.getDescription());
return service;
}
// Ensure deleted service does not exist
HttpResponseException exception = assertThrows(HttpResponseException.class, () -> getService(id, authHeaders));
TestUtils.assertResponse(exception, NOT_FOUND,
CatalogExceptionMessage.entityNotFound(Entity.STORAGE_SERVICE, id));
private static void validateService(StorageService service, String expectedName, String expectedDescription) {
assertNotNull(service.getId());
assertNotNull(service.getHref());
assertEquals(expectedName, service.getName());
assertEquals(expectedDescription, service.getDescription());
}
// Ensure deleted service does not exist when getting by name
exception = assertThrows(HttpResponseException.class, () -> getServiceByName(name, null, authHeaders));
TestUtils.assertResponse(exception, NOT_FOUND,
CatalogExceptionMessage.entityNotFound(Entity.STORAGE_SERVICE, name));
}
public static StorageService getService(UUID id, Map<String, String> authHeaders) throws HttpResponseException {
return getService(id, null, authHeaders);
}
public static StorageService getService(UUID id, String fields, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("services/storageServices/" + id);
target = fields != null ? target.queryParam("fields", fields) : target;
return TestUtils.get(target, StorageService.class, authHeaders);
}
public static StorageService getServiceByName(String name, String fields, Map<String, String> authHeaders)
throws HttpResponseException {
WebTarget target = CatalogApplicationTest.getResource("services/storageServices/name/" + name);
target = fields != null ? target.queryParam("fields", fields) : target;
return TestUtils.get(target, StorageService.class, authHeaders);
}
public static StorageService updateStorageService(CreateStorageService updated,
Response.Status status, Map<String, String> authHeaders)
throws HttpResponseException {
return TestUtils.put(CatalogApplicationTest.getResource("services/storageServices"), updated,
StorageService.class, status, authHeaders);
}
public static void updateAndCheckService(CreateStorageService update, Response.Status status,
Map<String, String> authHeaders) throws HttpResponseException {
StorageService service = updateStorageService(update, status, authHeaders);
validateService(service, service.getName(), update.getDescription());
// GET the newly updated storage and validate
StorageService getService = getService(service.getId(), authHeaders);
validateService(getService, service.getName(), update.getDescription());
// GET the newly updated storage by name and validate
getService = getServiceByName(service.getName(), null, authHeaders);
validateService(getService, service.getName(), update.getDescription());
}
private void deleteService(UUID id, String name, Map<String, String> authHeaders) throws HttpResponseException {
TestUtils.delete(CatalogApplicationTest.getResource("services/storageServices/" + id), authHeaders);
// Ensure deleted service does not exist
HttpResponseException exception = assertThrows(HttpResponseException.class, () -> getService(id, authHeaders));
TestUtils.assertResponse(exception, NOT_FOUND,
CatalogExceptionMessage.entityNotFound(Entity.STORAGE_SERVICE, id));
// Ensure deleted service does not exist when getting by name
exception = assertThrows(HttpResponseException.class, () -> getServiceByName(name, null, authHeaders));
TestUtils.assertResponse(exception, NOT_FOUND,
CatalogExceptionMessage.entityNotFound(Entity.STORAGE_SERVICE, name));
}
@Override
public Object createRequest(TestInfo test, int index, String description, String displayName,
EntityReference owner) throws URISyntaxException {
return create(test, index).withDescription(description);
}
@Override
public void validateCreatedEntity(StorageService createdEntity, Object request, Map<String, String> authHeaders) throws HttpResponseException {
}
@Override
public void validateUpdatedEntity(StorageService updatedEntity, Object request, Map<String, String> authHeaders) throws HttpResponseException {
}
@Override
public void compareEntities(StorageService expected, StorageService updated, Map<String, String> authHeaders) throws HttpResponseException {
}
@Override
public EntityInterface<StorageService> getEntityInterface(StorageService entity) {
return new StorageServiceEntityInterface(entity);
}
@Override
public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException {
}
}