diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java index e8d1a77876a..94282bd4328 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java @@ -28,8 +28,6 @@ import org.openmetadata.catalog.util.EntityInterface; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.JsonUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URI; @@ -42,7 +40,6 @@ import java.util.UUID; import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound; public class IngestionRepository extends EntityRepository { - private static final Logger LOG = LoggerFactory.getLogger(IngestionRepository.class); private static final Fields INGESTION_UPDATE_FIELDS = new Fields(IngestionResource.FIELD_LIST, "owner,tags"); private static final Fields INGESTION_PATCH_FIELDS = new Fields(IngestionResource.FIELD_LIST, @@ -50,7 +47,7 @@ public class IngestionRepository extends EntityRepository { private final CollectionDAO dao; public IngestionRepository(CollectionDAO dao) { - super(Ingestion.class, dao.ingestionDAO(), dao, INGESTION_PATCH_FIELDS, INGESTION_UPDATE_FIELDS); + super(Entity.INGESTION, Ingestion.class, dao.ingestionDAO(), dao, INGESTION_PATCH_FIELDS, INGESTION_UPDATE_FIELDS); this.dao = dao; } @@ -269,6 +266,9 @@ public class IngestionRepository extends EntityRepository { @Override public void setOwner(EntityReference owner) { entity.setOwner(owner); } + @Override + public Ingestion withHref(URI href) { return entity.withHref(href); } + @Override public void setTags(List tags) { entity.setTags(tags); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java index 1d1ed127446..dc1ce63ab24 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java @@ -421,7 +421,7 @@ public class TableResource { SecurityUtil.checkAdminOrBotRole(authorizer, securityContext); Fields fields = new Fields(FIELD_LIST, "tableQueries"); dao.addQuery(UUID.fromString(id), sqlQuery); - Table table = dao.get(id, fields); + Table table = dao.get(uriInfo, id, fields); return addHref(uriInfo, table); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/IngestionResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/IngestionResource.java index 531d8790b8c..2b32e8295ff 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/IngestionResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/IngestionResource.java @@ -27,6 +27,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import org.openmetadata.catalog.CatalogApplicationConfig; +import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.api.operations.workflows.CreateIngestion; import org.openmetadata.catalog.ingestion.AirflowRESTClient; import org.openmetadata.catalog.jdbi3.CollectionDAO; @@ -37,7 +38,6 @@ import org.openmetadata.catalog.security.CatalogAuthorizer; import org.openmetadata.catalog.security.SecurityUtil; import org.openmetadata.catalog.type.EntityHistory; import org.openmetadata.catalog.type.EntityReference; -import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; import org.openmetadata.catalog.util.RestUtil; import org.openmetadata.catalog.util.RestUtil.PutResponse; @@ -68,11 +68,9 @@ import java.io.UnsupportedEncodingException; import java.security.GeneralSecurityException; import java.text.ParseException; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.UUID; @Path("operations/v1/ingestion") @@ -91,15 +89,10 @@ public class IngestionResource { ref.withHref(RestUtil.getHref(uriInfo, INGESTION_COLLECTION_PATH, ref.getId())); } - public static List addHref(UriInfo uriInfo, List ingestions) { - Optional.ofNullable(ingestions).orElse(Collections.emptyList()).forEach(i -> addHref(uriInfo, i)); - return ingestions; - } - public static Ingestion addHref(UriInfo uriInfo, Ingestion ingestion) { ingestion.setHref(RestUtil.getHref(uriInfo, INGESTION_COLLECTION_PATH, ingestion.getId())); - EntityUtil.addHref(uriInfo, ingestion.getOwner()); - EntityUtil.addHref(uriInfo, ingestion.getService()); + Entity.withHref(uriInfo, ingestion.getOwner()); + Entity.withHref(uriInfo, ingestion.getService()); return ingestion; } @@ -165,11 +158,10 @@ public class IngestionResource { ResultList ingestions; if (before != null) { // Reverse paging - ingestions = dao.listBefore(fields, null, limitParam, before); // Ask for one extra entry + ingestions = dao.listBefore(uriInfo, fields, null, limitParam, before); // Ask for one extra entry } else { // Forward paging or first page - ingestions = dao.listAfter(fields, null, limitParam, after); + ingestions = dao.listAfter(uriInfo, fields, null, limitParam, after); } - addHref(uriInfo, ingestions.getData()); return ingestions; } @@ -206,7 +198,7 @@ public class IngestionResource { schema = @Schema(type = "string", example = FIELDS)) @QueryParam("fields") String fieldsParam) throws IOException, ParseException { Fields fields = new Fields(FIELD_LIST, fieldsParam); - return addHref(uriInfo, dao.get(id, fields)); + return dao.get(uriInfo, id, fields); } @GET @@ -246,8 +238,7 @@ public class IngestionResource { schema = @Schema(type = "string", example = FIELDS)) @QueryParam("fields") String fieldsParam) throws IOException, ParseException { Fields fields = new Fields(FIELD_LIST, fieldsParam); - Ingestion ingestion = dao.getByName(fqn, fields); - return addHref(uriInfo, ingestion); + return dao.getByName(uriInfo, fqn, fields); } @@ -266,7 +257,7 @@ public class IngestionResource { Ingestion ingestion = getIngestion(securityContext, create); deploy(ingestion); // write to db only when the deployment is successful - ingestion = addHref(uriInfo, dao.create(ingestion)); + ingestion = dao.create(uriInfo, ingestion); return Response.created(ingestion.getHref()).entity(ingestion).build(); } @@ -288,11 +279,10 @@ public class IngestionResource { "]")})) JsonPatch patch) throws IOException, ParseException { Fields fields = new Fields(FIELD_LIST, FIELDS); - Ingestion ingestion = dao.get(id, fields); + Ingestion ingestion = dao.get(uriInfo, id, fields); SecurityUtil.checkAdminRoleOrPermissions(authorizer, securityContext, dao.getOwnerReference(ingestion)); - ingestion = dao.patch(UUID.fromString(id), securityContext.getUserPrincipal().getName(), patch); - return addHref(uriInfo, ingestion); + return dao.patch(uriInfo, UUID.fromString(id), securityContext.getUserPrincipal().getName(), patch); } @PUT @@ -310,9 +300,8 @@ public class IngestionResource { Ingestion ingestion = getIngestion(securityContext, create); deploy(ingestion); // write to db only when the deployment is successful - PutResponse response = dao.createOrUpdate(ingestion); - ingestion = addHref(uriInfo, response.getEntity()); - return Response.status(response.getStatus()).entity(ingestion).build(); + PutResponse response = dao.createOrUpdate(uriInfo, ingestion); + return response.toResponse(); } @POST @@ -328,9 +317,9 @@ public class IngestionResource { public Ingestion triggerIngestion(@Context UriInfo uriInfo, @PathParam("id") String id, @Context SecurityContext securityContext) throws IOException, ParseException { Fields fields = new Fields(FIELD_LIST, ""); - Ingestion ingestion = dao.get(id, fields); + Ingestion ingestion = dao.get(uriInfo, id, fields); airflowRESTClient.runPipeline(ingestion.getName()); - return addHref(uriInfo, dao.get(id, fields)); + return dao.get(uriInfo, id, fields); } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java index 4dc1805cda7..7f07da8ce0c 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java @@ -630,16 +630,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { validateUpdatedEntity(updated, request, authHeaders); validateChangeDescription(updated, updateType, changeDescription); validateEntityHistory(entityInterface.getId(), updateType, changeDescription, authHeaders); - - // GET ../entity/{id}/versions/{versionId} to get specific versions of the entity - // Get the latest version of the entity from the versions API and ensure it is correct - latestVersion = getVersion(entityInterface.getId(), entityInterface.getVersion(), authHeaders); - validateChangeDescription(latestVersion, updateType, changeDescription); - if (updateType != NO_CHANGE && updateType != UpdateType.CREATED){ - // Get the previous version of the entity from the versions API and ensure it is correct - T previousVersion = getVersion(entityInterface.getId(), changeDescription.getPreviousVersion(), authHeaders); - assertEquals(changeDescription.getPreviousVersion(), getEntityInterface(previousVersion).getVersion()); - } + validateLatestVersion(entityInterface, updateType, changeDescription, authHeaders); // GET the newly updated entity and validate T getEntity = getEntity(entityInterface.getId(), authHeaders); @@ -667,6 +658,21 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { } } + private void validateLatestVersion(EntityInterface entityInterface, UpdateType updateType, + ChangeDescription expectedChangeDescription, + Map authHeaders) throws IOException { + // GET ../entity/{id}/versions/{versionId} to get specific versions of the entity + // Get the latest version of the entity from the versions API and ensure it is correct + T latestVersion = getVersion(entityInterface.getId(), entityInterface.getVersion(), authHeaders); + validateChangeDescription(latestVersion, updateType, expectedChangeDescription); + if (updateType != NO_CHANGE && updateType != UpdateType.CREATED) { + // Get the previous version of the entity from the versions API and ensure it is correct + T previousVersion = getVersion(entityInterface.getId(), expectedChangeDescription.getPreviousVersion(), authHeaders); + assertEquals(expectedChangeDescription.getPreviousVersion(), getEntityInterface(previousVersion).getVersion()); + } + } + + protected final T patchEntityAndCheck(T updated, String originalJson, Map authHeaders, UpdateType updateType, ChangeDescription expectedChange) throws IOException { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/operations/IngestionResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/operations/IngestionResourceTest.java index e1ca3fa7a41..e1a9fd204c7 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/operations/IngestionResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/operations/IngestionResourceTest.java @@ -62,7 +62,7 @@ public class IngestionResourceTest extends EntityResourceTest { public static ConnectorConfig INGESTION_CONFIG; public IngestionResourceTest() { - super(Ingestion.class, IngestionResource.IngestionList.class, "ingestion", + super(Entity.INGESTION, Ingestion.class, IngestionResource.IngestionList.class, "ingestion", IngestionResource.FIELDS, false, true, true); } @@ -99,7 +99,7 @@ public class IngestionResourceTest extends EntityResourceTest { } @Override - public void validatePatchedEntity(Ingestion expected, Ingestion updated, Map authHeaders) + public void compareEntities(Ingestion expected, Ingestion updated, Map authHeaders) throws HttpResponseException { validateCommonEntityFields(getEntityInterface(updated), expected.getDescription(), TestUtils.getPrincipal(authHeaders), expected.getOwner()); @@ -120,7 +120,6 @@ public class IngestionResourceTest extends EntityResourceTest { return; } assertCommonFieldChange(fieldName, expected, actual); - }