diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ServiceEntityResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ServiceEntityResource.java new file mode 100644 index 00000000000..bcf928e761f --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ServiceEntityResource.java @@ -0,0 +1,48 @@ +package org.openmetadata.catalog.resources.services; + +import java.util.Collections; +import java.util.Optional; +import javax.ws.rs.core.SecurityContext; +import org.openmetadata.catalog.ServiceConnectionEntityInterface; +import org.openmetadata.catalog.ServiceEntityInterface; +import org.openmetadata.catalog.jdbi3.ServiceRepository; +import org.openmetadata.catalog.resources.EntityResource; +import org.openmetadata.catalog.secrets.SecretsManager; +import org.openmetadata.catalog.security.AuthorizationException; +import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.util.ResultList; + +public abstract class ServiceEntityResource< + T extends ServiceEntityInterface, R extends ServiceRepository, S extends ServiceConnectionEntityInterface> + extends EntityResource { + + private final SecretsManager secretsManager; + + protected ServiceEntityResource( + Class entityClass, R serviceRepository, Authorizer authorizer, SecretsManager secretsManager) { + super(entityClass, serviceRepository, authorizer); + this.secretsManager = secretsManager; + } + + protected T decryptOrNullify(SecurityContext securityContext, T service) { + try { + authorizer.authorizeAdmin(securityContext, secretsManager.isLocal()); + } catch (AuthorizationException e) { + return nullifyConnection(service); + } + secretsManager.encryptOrDecryptServiceConnection( + service.getConnection(), extractServiceType(service), service.getName(), false); + return service; + } + + protected ResultList decryptOrNullify(SecurityContext securityContext, ResultList services) { + Optional.ofNullable(services.getData()) + .orElse(Collections.emptyList()) + .forEach(mlModelService -> decryptOrNullify(securityContext, mlModelService)); + return services; + } + + protected abstract T nullifyConnection(T service); + + protected abstract String extractServiceType(T service); +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/dashboard/DashboardServiceResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/dashboard/DashboardServiceResource.java index ae4a7099ddb..c10617c8cb9 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/dashboard/DashboardServiceResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/dashboard/DashboardServiceResource.java @@ -22,9 +22,7 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import java.io.IOException; -import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import javax.validation.Valid; import javax.validation.constraints.Max; @@ -51,10 +49,10 @@ import org.openmetadata.catalog.jdbi3.CollectionDAO; import org.openmetadata.catalog.jdbi3.DashboardServiceRepository; import org.openmetadata.catalog.jdbi3.ListFilter; import org.openmetadata.catalog.resources.Collection; -import org.openmetadata.catalog.resources.EntityResource; +import org.openmetadata.catalog.resources.services.ServiceEntityResource; import org.openmetadata.catalog.secrets.SecretsManager; -import org.openmetadata.catalog.security.AuthorizationException; import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.type.DashboardConnection; import org.openmetadata.catalog.type.EntityHistory; import org.openmetadata.catalog.type.Include; import org.openmetadata.catalog.util.JsonUtils; @@ -66,12 +64,11 @@ import org.openmetadata.catalog.util.ResultList; @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "dashboardServices") -public class DashboardServiceResource extends EntityResource { +public class DashboardServiceResource + extends ServiceEntityResource { public static final String COLLECTION_PATH = "v1/services/dashboardServices"; static final String FIELDS = FIELD_OWNER; - private final SecretsManager secretsManager; - @Override public DashboardService addHref(UriInfo uriInfo, DashboardService service) { service.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, service.getId())); @@ -80,8 +77,7 @@ public class DashboardServiceResource extends EntityResource { @@ -356,22 +352,13 @@ public class DashboardServiceResource extends EntityResource decryptOrNullify( - SecurityContext securityContext, ResultList dashboardServices) { - Optional.ofNullable(dashboardServices.getData()) - .orElse(Collections.emptyList()) - .forEach(dashboardService -> decryptOrNullify(securityContext, dashboardService)); - return dashboardServices; + @Override + protected DashboardService nullifyConnection(DashboardService service) { + return service.withConnection(null); } - private DashboardService decryptOrNullify(SecurityContext securityContext, DashboardService dashboardService) { - try { - authorizer.authorizeAdmin(securityContext, true); - } catch (AuthorizationException e) { - return dashboardService.withConnection(null); - } - secretsManager.encryptOrDecryptServiceConnection( - dashboardService.getConnection(), dashboardService.getServiceType().value(), dashboardService.getName(), false); - return dashboardService; + @Override + protected String extractServiceType(DashboardService service) { + return service.getServiceType().value(); } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/database/DatabaseServiceResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/database/DatabaseServiceResource.java index db20c67d094..ce1a572cb70 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/database/DatabaseServiceResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/database/DatabaseServiceResource.java @@ -20,9 +20,7 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import java.io.IOException; -import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import javax.validation.Valid; import javax.validation.constraints.Max; @@ -45,14 +43,14 @@ import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.api.services.CreateDatabaseService; +import org.openmetadata.catalog.api.services.DatabaseConnection; import org.openmetadata.catalog.entity.services.DatabaseService; import org.openmetadata.catalog.jdbi3.CollectionDAO; import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository; import org.openmetadata.catalog.jdbi3.ListFilter; import org.openmetadata.catalog.resources.Collection; -import org.openmetadata.catalog.resources.EntityResource; +import org.openmetadata.catalog.resources.services.ServiceEntityResource; import org.openmetadata.catalog.secrets.SecretsManager; -import org.openmetadata.catalog.security.AuthorizationException; import org.openmetadata.catalog.security.Authorizer; import org.openmetadata.catalog.type.EntityHistory; import org.openmetadata.catalog.type.Include; @@ -67,12 +65,11 @@ import org.openmetadata.catalog.util.ResultList; @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "databaseServices") -public class DatabaseServiceResource extends EntityResource { +public class DatabaseServiceResource + extends ServiceEntityResource { public static final String COLLECTION_PATH = "v1/services/databaseServices/"; static final String FIELDS = "pipelines,owner"; - private final SecretsManager secretsManager; - @Override public DatabaseService addHref(UriInfo uriInfo, DatabaseService service) { service.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, service.getId())); @@ -82,8 +79,7 @@ public class DatabaseServiceResource extends EntityResource { @@ -362,22 +358,13 @@ public class DatabaseServiceResource extends EntityResource decryptOrNullify( - SecurityContext securityContext, ResultList databaseServices) { - Optional.ofNullable(databaseServices.getData()) - .orElse(Collections.emptyList()) - .forEach(databaseService -> decryptOrNullify(securityContext, databaseService)); - return databaseServices; + @Override + protected DatabaseService nullifyConnection(DatabaseService service) { + return service.withConnection(null); } - private DatabaseService decryptOrNullify(SecurityContext securityContext, DatabaseService databaseService) { - try { - authorizer.authorizeAdmin(securityContext, true); - } catch (AuthorizationException e) { - return databaseService.withConnection(null); - } - secretsManager.encryptOrDecryptServiceConnection( - databaseService.getConnection(), databaseService.getServiceType().value(), databaseService.getName(), false); - return databaseService; + @Override + protected String extractServiceType(DatabaseService service) { + return service.getServiceType().value(); } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/messaging/MessagingServiceResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/messaging/MessagingServiceResource.java index 225b4c95d20..ffe418d185d 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/messaging/MessagingServiceResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/messaging/MessagingServiceResource.java @@ -22,9 +22,7 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import java.io.IOException; -import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import javax.validation.Valid; import javax.validation.constraints.Max; @@ -51,12 +49,12 @@ import org.openmetadata.catalog.jdbi3.CollectionDAO; import org.openmetadata.catalog.jdbi3.ListFilter; import org.openmetadata.catalog.jdbi3.MessagingServiceRepository; import org.openmetadata.catalog.resources.Collection; -import org.openmetadata.catalog.resources.EntityResource; +import org.openmetadata.catalog.resources.services.ServiceEntityResource; import org.openmetadata.catalog.secrets.SecretsManager; -import org.openmetadata.catalog.security.AuthorizationException; import org.openmetadata.catalog.security.Authorizer; import org.openmetadata.catalog.type.EntityHistory; import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.type.MessagingConnection; import org.openmetadata.catalog.util.JsonUtils; import org.openmetadata.catalog.util.RestUtil; import org.openmetadata.catalog.util.ResultList; @@ -66,13 +64,12 @@ import org.openmetadata.catalog.util.ResultList; @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "messagingServices") -public class MessagingServiceResource extends EntityResource { +public class MessagingServiceResource + extends ServiceEntityResource { public static final String COLLECTION_PATH = "v1/services/messagingServices/"; public static final String FIELDS = FIELD_OWNER; - private final SecretsManager secretsManager; - @Override public MessagingService addHref(UriInfo uriInfo, MessagingService service) { service.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, service.getId())); @@ -81,8 +78,7 @@ public class MessagingServiceResource extends EntityResource { @@ -296,7 +292,9 @@ public class MessagingServiceResource extends EntityResource decryptOrNullify( - SecurityContext securityContext, ResultList messagingServices) { - Optional.ofNullable(messagingServices.getData()) - .orElse(Collections.emptyList()) - .forEach(messagingService -> decryptOrNullify(securityContext, messagingService)); - return messagingServices; + @Override + protected MessagingService nullifyConnection(MessagingService service) { + return service.withConnection(null); } - private MessagingService decryptOrNullify(SecurityContext securityContext, MessagingService messagingService) { - try { - authorizer.authorizeAdmin(securityContext, true); - } catch (AuthorizationException e) { - return messagingService.withConnection(null); - } - secretsManager.encryptOrDecryptServiceConnection( - messagingService.getConnection(), messagingService.getServiceType().value(), messagingService.getName(), false); - return messagingService; + @Override + protected String extractServiceType(MessagingService service) { + return service.getServiceType().value(); } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/mlmodel/MlModelServiceResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/mlmodel/MlModelServiceResource.java index 5b8198e6051..7eb7cea605c 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/mlmodel/MlModelServiceResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/mlmodel/MlModelServiceResource.java @@ -20,9 +20,7 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import java.io.IOException; -import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import javax.validation.Valid; import javax.validation.constraints.Max; @@ -49,12 +47,12 @@ import org.openmetadata.catalog.jdbi3.CollectionDAO; import org.openmetadata.catalog.jdbi3.ListFilter; import org.openmetadata.catalog.jdbi3.MlModelServiceRepository; import org.openmetadata.catalog.resources.Collection; -import org.openmetadata.catalog.resources.EntityResource; +import org.openmetadata.catalog.resources.services.ServiceEntityResource; import org.openmetadata.catalog.secrets.SecretsManager; -import org.openmetadata.catalog.security.AuthorizationException; import org.openmetadata.catalog.security.Authorizer; import org.openmetadata.catalog.type.EntityHistory; import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.type.MlModelConnection; import org.openmetadata.catalog.util.JsonUtils; import org.openmetadata.catalog.util.RestUtil; import org.openmetadata.catalog.util.ResultList; @@ -64,13 +62,12 @@ import org.openmetadata.catalog.util.ResultList; @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "mlmodelServices") -public class MlModelServiceResource extends EntityResource { +public class MlModelServiceResource + extends ServiceEntityResource { public static final String COLLECTION_PATH = "v1/services/mlmodelServices/"; public static final String FIELDS = "pipelines,owner"; - private final SecretsManager secretsManager; - @Override public MlModelService addHref(UriInfo uriInfo, MlModelService service) { service.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, service.getId())); @@ -80,8 +77,7 @@ public class MlModelServiceResource extends EntityResource { @@ -295,7 +291,9 @@ public class MlModelServiceResource extends EntityResource decryptOrNullify( - SecurityContext securityContext, ResultList mlModelServices) { - Optional.ofNullable(mlModelServices.getData()) - .orElse(Collections.emptyList()) - .forEach(mlModelService -> decryptOrNullify(securityContext, mlModelService)); - return mlModelServices; + @Override + protected MlModelService nullifyConnection(MlModelService service) { + return service.withConnection(null); } - private MlModelService decryptOrNullify(SecurityContext securityContext, MlModelService mlModelService) { - try { - authorizer.authorizeAdmin(securityContext, true); - } catch (AuthorizationException e) { - return mlModelService.withConnection(null); - } - secretsManager.encryptOrDecryptServiceConnection( - mlModelService.getConnection(), mlModelService.getServiceType().value(), mlModelService.getName(), false); - return mlModelService; + @Override + protected String extractServiceType(MlModelService service) { + return service.getServiceType().value(); } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/pipeline/PipelineServiceResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/pipeline/PipelineServiceResource.java index e977c57028d..d5d57990ca3 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/pipeline/PipelineServiceResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/pipeline/PipelineServiceResource.java @@ -20,9 +20,7 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import java.io.IOException; -import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import javax.validation.Valid; import javax.validation.constraints.Max; @@ -49,12 +47,12 @@ import org.openmetadata.catalog.jdbi3.CollectionDAO; import org.openmetadata.catalog.jdbi3.ListFilter; import org.openmetadata.catalog.jdbi3.PipelineServiceRepository; import org.openmetadata.catalog.resources.Collection; -import org.openmetadata.catalog.resources.EntityResource; +import org.openmetadata.catalog.resources.services.ServiceEntityResource; import org.openmetadata.catalog.secrets.SecretsManager; -import org.openmetadata.catalog.security.AuthorizationException; import org.openmetadata.catalog.security.Authorizer; import org.openmetadata.catalog.type.EntityHistory; import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.type.PipelineConnection; import org.openmetadata.catalog.util.JsonUtils; import org.openmetadata.catalog.util.RestUtil; import org.openmetadata.catalog.util.ResultList; @@ -64,12 +62,11 @@ import org.openmetadata.catalog.util.ResultList; @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "pipelineServices") -public class PipelineServiceResource extends EntityResource { +public class PipelineServiceResource + extends ServiceEntityResource { public static final String COLLECTION_PATH = "v1/services/pipelineServices/"; static final String FIELDS = "pipelines,owner"; - private final SecretsManager secretsManager; - @Override public PipelineService addHref(UriInfo uriInfo, PipelineService service) { service.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, service.getId())); @@ -79,8 +76,7 @@ public class PipelineServiceResource extends EntityResource { @@ -357,22 +353,13 @@ public class PipelineServiceResource extends EntityResource decryptOrNullify( - SecurityContext securityContext, ResultList pipelineServices) { - Optional.ofNullable(pipelineServices.getData()) - .orElse(Collections.emptyList()) - .forEach(pipelineService -> decryptOrNullify(securityContext, pipelineService)); - return pipelineServices; + @Override + protected PipelineService nullifyConnection(PipelineService service) { + return service.withConnection(null); } - private PipelineService decryptOrNullify(SecurityContext securityContext, PipelineService pipelineService) { - try { - authorizer.authorizeAdmin(securityContext, true); - } catch (AuthorizationException e) { - return pipelineService.withConnection(null); - } - secretsManager.encryptOrDecryptServiceConnection( - pipelineService.getConnection(), pipelineService.getServiceType().value(), pipelineService.getName(), false); - return pipelineService; + @Override + protected String extractServiceType(PipelineService service) { + return service.getServiceType().value(); } } diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json index 8370060bc56..713bfa75953 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json @@ -145,6 +145,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false } diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json index db61d246879..a324f5a3be5 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json @@ -284,6 +284,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false } diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json index b8404f0f28b..aee7f294a8b 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json @@ -115,6 +115,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false } diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/metadataService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/metadataService.json index dfff3a257c6..1a6e034b549 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/metadataService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/metadataService.json @@ -104,6 +104,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false } diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/mlmodelService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/mlmodelService.json index 1ddbc56b2ab..f4f48f5ce18 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/mlmodelService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/mlmodelService.json @@ -107,6 +107,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false } diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json index 5da36840e94..7eeff97be5d 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json @@ -113,6 +113,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/DashboardServiceResourceUnitTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/DashboardServiceResourceUnitTest.java new file mode 100644 index 00000000000..ce86e893b13 --- /dev/null +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/DashboardServiceResourceUnitTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.services; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.openmetadata.catalog.Entity.FIELD_OWNER; + +import java.io.IOException; +import java.util.UUID; +import javax.ws.rs.core.UriInfo; +import org.openmetadata.catalog.api.services.CreateDashboardService; +import org.openmetadata.catalog.entity.services.DashboardService; +import org.openmetadata.catalog.jdbi3.CollectionDAO; +import org.openmetadata.catalog.jdbi3.DashboardServiceRepository; +import org.openmetadata.catalog.resources.services.dashboard.DashboardServiceResource; +import org.openmetadata.catalog.secrets.SecretsManager; +import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.type.DashboardConnection; +import org.openmetadata.catalog.type.Include; + +public class DashboardServiceResourceUnitTest + extends ServiceResourceTest< + DashboardServiceResource, DashboardService, DashboardServiceRepository, DashboardConnection> { + + @Override + protected DashboardServiceResource newServiceResource( + CollectionDAO collectionDAO, Authorizer authorizer, SecretsManager secretsManager) { + return new DashboardServiceResource(collectionDAO, authorizer, secretsManager); + } + + @Override + protected void mockServiceResourceSpecific() throws IOException { + service = mock(DashboardService.class); + CollectionDAO.DashboardServiceDAO entityDAO = mock(CollectionDAO.DashboardServiceDAO.class); + when(collectionDAO.dashboardServiceDAO()).thenReturn(entityDAO); + lenient().when(service.getServiceType()).thenReturn(CreateDashboardService.DashboardServiceType.Tableau); + lenient().when(service.getConnection()).thenReturn(mock(DashboardConnection.class)); + lenient().when(service.withConnection(isNull())).thenReturn(service); + when(entityDAO.findEntityById(any(), any())).thenReturn(service); + when(entityDAO.getEntityClass()).thenReturn(DashboardService.class); + } + + @Override + protected String serviceType() { + return CreateDashboardService.DashboardServiceType.Tableau.value(); + } + + @Override + protected void verifyServiceWithConnectionCall(boolean shouldBeNull, DashboardService service) { + verify(service, times(shouldBeNull ? 1 : 0)).withConnection(isNull()); + } + + @Override + protected DashboardService callGetFromResource(DashboardServiceResource resource) throws IOException { + return resource.get(mock(UriInfo.class), securityContext, UUID.randomUUID().toString(), FIELD_OWNER, Include.ALL); + } +} diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/DatabaseServiceResourceUnitTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/DatabaseServiceResourceUnitTest.java new file mode 100644 index 00000000000..e953bed7422 --- /dev/null +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/DatabaseServiceResourceUnitTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.services; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.openmetadata.catalog.Entity.FIELD_OWNER; + +import java.io.IOException; +import java.util.UUID; +import javax.ws.rs.core.UriInfo; +import org.openmetadata.catalog.api.services.CreateDatabaseService; +import org.openmetadata.catalog.api.services.DatabaseConnection; +import org.openmetadata.catalog.entity.services.DatabaseService; +import org.openmetadata.catalog.jdbi3.CollectionDAO; +import org.openmetadata.catalog.jdbi3.DatabaseServiceRepository; +import org.openmetadata.catalog.resources.services.database.DatabaseServiceResource; +import org.openmetadata.catalog.secrets.SecretsManager; +import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.type.Include; + +public class DatabaseServiceResourceUnitTest + extends ServiceResourceTest< + DatabaseServiceResource, DatabaseService, DatabaseServiceRepository, DatabaseConnection> { + + @Override + protected DatabaseServiceResource newServiceResource( + CollectionDAO collectionDAO, Authorizer authorizer, SecretsManager secretsManager) { + return new DatabaseServiceResource(collectionDAO, authorizer, secretsManager); + } + + @Override + protected void mockServiceResourceSpecific() throws IOException { + service = mock(DatabaseService.class); + CollectionDAO.DatabaseServiceDAO entityDAO = mock(CollectionDAO.DatabaseServiceDAO.class); + when(collectionDAO.dbServiceDAO()).thenReturn(entityDAO); + lenient().when(service.getServiceType()).thenReturn(CreateDatabaseService.DatabaseServiceType.Mysql); + lenient().when(service.getConnection()).thenReturn(mock(DatabaseConnection.class)); + lenient().when(service.withConnection(isNull())).thenReturn(service); + when(entityDAO.findEntityById(any(), any())).thenReturn(service); + when(entityDAO.getEntityClass()).thenReturn(DatabaseService.class); + } + + @Override + protected String serviceType() { + return CreateDatabaseService.DatabaseServiceType.Mysql.value(); + } + + @Override + protected void verifyServiceWithConnectionCall(boolean shouldBeNull, DatabaseService service) { + verify(service, times(shouldBeNull ? 1 : 0)).withConnection(isNull()); + } + + @Override + protected DatabaseService callGetFromResource(DatabaseServiceResource resource) throws IOException { + return resource.get(mock(UriInfo.class), securityContext, UUID.randomUUID().toString(), FIELD_OWNER, Include.ALL); + } +} diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/MessagingServiceResourceUnitTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/MessagingServiceResourceUnitTest.java new file mode 100644 index 00000000000..ca6a4ea5c10 --- /dev/null +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/MessagingServiceResourceUnitTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.services; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.openmetadata.catalog.Entity.FIELD_OWNER; + +import java.io.IOException; +import java.util.UUID; +import javax.ws.rs.core.UriInfo; +import org.openmetadata.catalog.api.services.CreateMessagingService; +import org.openmetadata.catalog.entity.services.MessagingService; +import org.openmetadata.catalog.jdbi3.CollectionDAO; +import org.openmetadata.catalog.jdbi3.MessagingServiceRepository; +import org.openmetadata.catalog.resources.services.messaging.MessagingServiceResource; +import org.openmetadata.catalog.secrets.SecretsManager; +import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.type.MessagingConnection; + +public class MessagingServiceResourceUnitTest + extends ServiceResourceTest< + MessagingServiceResource, MessagingService, MessagingServiceRepository, MessagingConnection> { + + @Override + protected MessagingServiceResource newServiceResource( + CollectionDAO collectionDAO, Authorizer authorizer, SecretsManager secretsManager) { + return new MessagingServiceResource(collectionDAO, authorizer, secretsManager); + } + + @Override + protected void mockServiceResourceSpecific() throws IOException { + service = mock(MessagingService.class); + CollectionDAO.MessagingServiceDAO entityDAO = mock(CollectionDAO.MessagingServiceDAO.class); + when(collectionDAO.messagingServiceDAO()).thenReturn(entityDAO); + lenient().when(service.getServiceType()).thenReturn(CreateMessagingService.MessagingServiceType.Kafka); + lenient().when(service.getConnection()).thenReturn(mock(MessagingConnection.class)); + lenient().when(service.withConnection(isNull())).thenReturn(service); + when(entityDAO.findEntityById(any(), any())).thenReturn(service); + when(entityDAO.getEntityClass()).thenReturn(MessagingService.class); + } + + @Override + protected String serviceType() { + return CreateMessagingService.MessagingServiceType.Kafka.value(); + } + + @Override + protected void verifyServiceWithConnectionCall(boolean shouldBeNull, MessagingService service) { + verify(service, times(shouldBeNull ? 1 : 0)).withConnection(isNull()); + } + + @Override + protected MessagingService callGetFromResource(MessagingServiceResource resource) throws IOException { + return resource.get(mock(UriInfo.class), securityContext, UUID.randomUUID().toString(), FIELD_OWNER, Include.ALL); + } +} diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/MlModelServiceResourceUnitTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/MlModelServiceResourceUnitTest.java new file mode 100644 index 00000000000..edb5c9e9269 --- /dev/null +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/MlModelServiceResourceUnitTest.java @@ -0,0 +1,76 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.services; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.openmetadata.catalog.Entity.FIELD_OWNER; + +import java.io.IOException; +import java.util.UUID; +import javax.ws.rs.core.UriInfo; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.openmetadata.catalog.api.services.CreateMlModelService; +import org.openmetadata.catalog.entity.services.MlModelService; +import org.openmetadata.catalog.jdbi3.CollectionDAO; +import org.openmetadata.catalog.jdbi3.MlModelServiceRepository; +import org.openmetadata.catalog.resources.services.mlmodel.MlModelServiceResource; +import org.openmetadata.catalog.secrets.SecretsManager; +import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.type.MlModelConnection; + +@ExtendWith(MockitoExtension.class) +public class MlModelServiceResourceUnitTest + extends ServiceResourceTest { + + @Override + protected MlModelServiceResource newServiceResource( + CollectionDAO collectionDAO, Authorizer authorizer, SecretsManager secretsManager) { + return new MlModelServiceResource(collectionDAO, authorizer, secretsManager); + } + + @Override + protected void mockServiceResourceSpecific() throws IOException { + service = mock(MlModelService.class); + CollectionDAO.MlModelServiceDAO entityDAO = mock(CollectionDAO.MlModelServiceDAO.class); + when(collectionDAO.mlModelServiceDAO()).thenReturn(entityDAO); + lenient().when(service.getServiceType()).thenReturn(CreateMlModelService.MlModelServiceType.Mlflow); + lenient().when(service.getConnection()).thenReturn(mock(MlModelConnection.class)); + lenient().when(service.withConnection(isNull())).thenReturn(service); + when(entityDAO.findEntityById(any(), any())).thenReturn(service); + when(entityDAO.getEntityClass()).thenReturn(MlModelService.class); + } + + @Override + protected String serviceType() { + return CreateMlModelService.MlModelServiceType.Mlflow.value(); + } + + @Override + protected void verifyServiceWithConnectionCall(boolean shouldBeNull, MlModelService service) { + verify(service, times(shouldBeNull ? 1 : 0)).withConnection(isNull()); + } + + @Override + protected MlModelService callGetFromResource(MlModelServiceResource resource) throws IOException { + return resource.get(mock(UriInfo.class), securityContext, UUID.randomUUID().toString(), FIELD_OWNER, Include.ALL); + } +} diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/PipelineServiceResourceUnitTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/PipelineServiceResourceUnitTest.java new file mode 100644 index 00000000000..d823f61ba39 --- /dev/null +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/PipelineServiceResourceUnitTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.services; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.openmetadata.catalog.Entity.FIELD_OWNER; + +import java.io.IOException; +import java.util.UUID; +import javax.ws.rs.core.UriInfo; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.openmetadata.catalog.api.services.CreatePipelineService; +import org.openmetadata.catalog.entity.services.PipelineService; +import org.openmetadata.catalog.jdbi3.CollectionDAO; +import org.openmetadata.catalog.jdbi3.PipelineServiceRepository; +import org.openmetadata.catalog.resources.services.pipeline.PipelineServiceResource; +import org.openmetadata.catalog.secrets.SecretsManager; +import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.type.PipelineConnection; + +@ExtendWith(MockitoExtension.class) +public class PipelineServiceResourceUnitTest + extends ServiceResourceTest< + PipelineServiceResource, PipelineService, PipelineServiceRepository, PipelineConnection> { + + @Override + protected PipelineServiceResource newServiceResource( + CollectionDAO collectionDAO, Authorizer authorizer, SecretsManager secretsManager) { + return new PipelineServiceResource(collectionDAO, authorizer, secretsManager); + } + + @Override + protected void mockServiceResourceSpecific() throws IOException { + service = mock(PipelineService.class); + CollectionDAO.PipelineServiceDAO entityDAO = mock(CollectionDAO.PipelineServiceDAO.class); + when(collectionDAO.pipelineServiceDAO()).thenReturn(entityDAO); + lenient().when(service.getServiceType()).thenReturn(CreatePipelineService.PipelineServiceType.Airflow); + lenient().when(service.getConnection()).thenReturn(mock(PipelineConnection.class)); + lenient().when(service.withConnection(isNull())).thenReturn(service); + when(entityDAO.findEntityById(any(), any())).thenReturn(service); + when(entityDAO.getEntityClass()).thenReturn(PipelineService.class); + } + + @Override + protected String serviceType() { + return CreatePipelineService.PipelineServiceType.Airflow.value(); + } + + @Override + protected void verifyServiceWithConnectionCall(boolean shouldBeNull, PipelineService service) { + verify(service, times(shouldBeNull ? 1 : 0)).withConnection(isNull()); + } + + @Override + protected PipelineService callGetFromResource(PipelineServiceResource resource) throws IOException { + return resource.get(mock(UriInfo.class), securityContext, UUID.randomUUID().toString(), FIELD_OWNER, Include.ALL); + } +} diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ServiceResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ServiceResourceTest.java new file mode 100644 index 00000000000..88dee463d29 --- /dev/null +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ServiceResourceTest.java @@ -0,0 +1,113 @@ +package org.openmetadata.catalog.resources.services; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.notNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.UUID; +import javax.ws.rs.core.SecurityContext; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.openmetadata.catalog.ServiceConnectionEntityInterface; +import org.openmetadata.catalog.ServiceEntityInterface; +import org.openmetadata.catalog.jdbi3.CollectionDAO; +import org.openmetadata.catalog.jdbi3.ServiceRepository; +import org.openmetadata.catalog.secrets.SecretsManager; +import org.openmetadata.catalog.security.AuthorizationException; +import org.openmetadata.catalog.security.Authorizer; + +@ExtendWith(MockitoExtension.class) +public abstract class ServiceResourceTest< + T extends ServiceEntityResource, + R extends ServiceEntityInterface, + S extends ServiceRepository, + U extends ServiceConnectionEntityInterface> { + + T serviceResource; + + @Mock protected CollectionDAO collectionDAO; + + @Mock protected Authorizer authorizer; + + @Mock protected SecretsManager secretsManager; + + protected R service; + + @Mock protected SecurityContext securityContext; + + @BeforeEach + void beforeEach() throws IOException { + mockServiceResourceSpecific(); + when(collectionDAO.relationshipDAO()).thenReturn(mock(CollectionDAO.EntityRelationshipDAO.class)); + when(service.getId()).thenReturn(UUID.randomUUID()); + when(service.withHref(any())).thenReturn(service); + lenient() + .when( + secretsManager.encryptOrDecryptServiceConnectionConfig( + any(), anyString(), anyString(), anyString(), anyBoolean())) + .thenReturn(service); + serviceResource = newServiceResource(collectionDAO, authorizer, secretsManager); + } + + protected abstract void mockServiceResourceSpecific() throws IOException; + + protected abstract T newServiceResource( + CollectionDAO collectionDAO, Authorizer authorizer, SecretsManager secretsManager); + + @ParameterizedTest + @CsvSource({ + "true, true, true, false", + "true, true, false, false", + "true, false, true, false", + "true, false, false, true", + "false, true, true, false", + "false, true, false, false", + "false, false, true, true", + "false, false, false, true" + }) + void testGetCallDecryptOrNullify(boolean isLocalSecretManager, boolean isAdmin, boolean isBot, boolean shouldBeNull) + throws IOException { + lenient().when(secretsManager.isLocal()).thenReturn(isLocalSecretManager); + + if (isLocalSecretManager && !isAdmin && !isBot) { + lenient() + .doThrow(new AuthorizationException("")) + .when(authorizer) + .authorizeAdmin(any(SecurityContext.class), eq(true)); + } else if (!isLocalSecretManager && !isAdmin) { + lenient() + .doThrow(new AuthorizationException("")) + .when(authorizer) + .authorizeAdmin(any(SecurityContext.class), eq(false)); + } + + R actual = callGetFromResource(serviceResource); + + verify(secretsManager, times(1)).isLocal(); + verify(secretsManager, times(shouldBeNull ? 0 : 1)) + .encryptOrDecryptServiceConnection(notNull(), eq(serviceType()), any(), eq(false)); + + verifyServiceWithConnectionCall(shouldBeNull, service); + + assertEquals(service, actual); + } + + protected abstract String serviceType(); + + protected abstract void verifyServiceWithConnectionCall(boolean shouldBeNull, R service); + + protected abstract R callGetFromResource(T resource) throws IOException; +} diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResourceUnitTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResourceUnitTest.java new file mode 100644 index 00000000000..75adaf510f9 --- /dev/null +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResourceUnitTest.java @@ -0,0 +1,95 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.resources.services.ingestionpipelines; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import javax.ws.rs.core.SecurityContext; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.MockedConstruction.Context; +import org.mockito.junit.jupiter.MockitoExtension; +import org.openmetadata.catalog.CatalogApplicationConfig; +import org.openmetadata.catalog.airflow.AirflowRESTClient; +import org.openmetadata.catalog.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.catalog.jdbi3.CollectionDAO; +import org.openmetadata.catalog.security.Authorizer; +import org.openmetadata.catalog.type.EntityReference; +import org.openmetadata.catalog.util.PipelineServiceClient; + +@ExtendWith(MockitoExtension.class) +public class IngestionPipelineResourceUnitTest { + + private static final String DAG_ID = UUID.randomUUID().toString(); + + private IngestionPipelineResource ingestionPipelineResource; + + @Mock SecurityContext securityContext; + + @Mock Authorizer authorizer; + + @Mock CollectionDAO collectionDAO; + + @Mock CatalogApplicationConfig catalogApplicationConfig; + + @Mock IngestionPipeline ingestionPipeline; + + @BeforeEach + void setUp() throws IOException { + CollectionDAO.IngestionPipelineDAO entityDAO = mock(CollectionDAO.IngestionPipelineDAO.class); + CollectionDAO.EntityRelationshipDAO relationshipDAO = mock(CollectionDAO.EntityRelationshipDAO.class); + CollectionDAO.EntityRelationshipRecord entityRelationshipRecord = + mock(CollectionDAO.EntityRelationshipRecord.class); + when(entityRelationshipRecord.getId()).thenReturn(UUID.randomUUID()); + when(entityRelationshipRecord.getType()).thenReturn("ingestionPipeline"); + when(relationshipDAO.findFrom(any(), any(), anyInt())).thenReturn(List.of(entityRelationshipRecord)); + when(collectionDAO.ingestionPipelineDAO()).thenReturn(entityDAO); + when(collectionDAO.relationshipDAO()).thenReturn(relationshipDAO); + when(entityDAO.findEntityById(any(), any())).thenReturn(ingestionPipeline); + when(entityDAO.findEntityReferenceById(any(), any())).thenReturn(mock(EntityReference.class)); + when(entityDAO.getEntityClass()).thenReturn(IngestionPipeline.class); + when(ingestionPipeline.getId()).thenReturn(UUID.fromString(DAG_ID)); + ingestionPipelineResource = new IngestionPipelineResource(collectionDAO, authorizer); + } + + @Test + public void testLastIngestionLogsAreRetrieved() throws IOException { + Map expectedMap = Map.of("task", "log"); + try (MockedConstruction mocked = + mockConstruction(AirflowRESTClient.class, this::preparePipelineServiceClient)) { + ingestionPipelineResource.initialize(catalogApplicationConfig); + assertEquals( + expectedMap, ingestionPipelineResource.getLastIngestionLogs(null, securityContext, DAG_ID).getEntity()); + PipelineServiceClient client = mocked.constructed().get(0); + verify(client).getLastIngestionLogs(ingestionPipeline); + } + } + + private void preparePipelineServiceClient(AirflowRESTClient mockPipelineServiceClient, Context context) { + when(mockPipelineServiceClient.getLastIngestionLogs(any())).thenReturn(Map.of("task", "log")); + } +} diff --git a/ingestion/src/metadata/utils/secrets_manager.py b/ingestion/src/metadata/utils/secrets_manager.py index 98f90a11d90..8439a9107c6 100644 --- a/ingestion/src/metadata/utils/secrets_manager.py +++ b/ingestion/src/metadata/utils/secrets_manager.py @@ -8,112 +8,112 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import inspect import json from abc import abstractmethod from pydoc import locate -from typing import Optional, Union +from typing import NewType, Optional, Union import boto3 from botocore.exceptions import ClientError +from pydantic.main import ModelMetaclass from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( SecretsManagerProvider, ) -from metadata.generated.schema.entity.services.dashboardService import DashboardService -from metadata.generated.schema.entity.services.databaseService import DatabaseService -from metadata.generated.schema.entity.services.messagingService import MessagingService -from metadata.generated.schema.entity.services.mlmodelService import MlModelService -from metadata.generated.schema.entity.services.pipelineService import PipelineService +from metadata.generated.schema.entity.services.connections.serviceConnection import ( + ServiceConnection, +) from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials from metadata.utils.logger import ingestion_logger from metadata.utils.singleton import Singleton logger = ingestion_logger() + SECRET_MANAGER_AIRFLOW_CONF = "openmetadata_secrets_manager" +ServiceConnectionType = NewType( + "ServiceConnectionType", ServiceConnection.__fields__["__root__"].type_ +) + class SecretsManager(metaclass=Singleton): @abstractmethod def add_service_config_connection( self, - service: Union[ - DashboardService, - DatabaseService, - MessagingService, - PipelineService, - MlModelService, - ], + service: ServiceConnectionType, service_type: str, - ) -> Union[ - DashboardService, - DatabaseService, - MessagingService, - PipelineService, - MlModelService, - ]: + ) -> ServiceConnectionType: pass @staticmethod + def to_service_simple(service_type: str) -> str: + return service_type.replace("Service", "").lower() + def build_secret_id( - service_type: str, service_connection_type: str, service_name: str + self, service_type: str, service_connection_type: str, service_name: str ) -> str: - return f"openmetadata-{service_type.lower()}-{service_connection_type.lower()}-{service_name.lower()}" + return f"openmetadata-{self.to_service_simple(service_type).lower()}-{service_connection_type.lower()}-{service_name.lower()}" + + def get_service_connection_class(self, service_type) -> ModelMetaclass: + service_conn_name = next( + ( + clazz[1] + for clazz in inspect.getmembers( + locate(f"metadata.generated.schema.entity.services.{service_type}"), + inspect.isclass, + ) + if clazz[0].lower() + == f"{self.to_service_simple(service_type)}connection" + ) + ).__name__ + service_conn_class = locate( + f"metadata.generated.schema.entity.services.{service_type}.{service_conn_name}" + ) + return service_conn_class + + def get_connection_class( + self, service_type: str, service_connection_type + ) -> ModelMetaclass: + connection_py_file = ( + service_connection_type[0].lower() + service_connection_type[1:] + ) + conn_class = locate( + f"metadata.generated.schema.entity.services.connections.{self.to_service_simple(service_type)}.{connection_py_file}Connection.{service_connection_type}Connection" + ) + return conn_class class LocalSecretsManager(SecretsManager): def add_service_config_connection( self, - service: Union[ - DashboardService, - DatabaseService, - MessagingService, - PipelineService, - MlModelService, - ], + service: ServiceConnectionType, service_type: str, - ) -> Union[ - DashboardService, - DatabaseService, - MessagingService, - PipelineService, - MlModelService, - ]: + ) -> ServiceConnectionType: return service class AWSSecretsManager(SecretsManager): def add_service_config_connection( self, - service: Union[ - DashboardService, - DatabaseService, - MessagingService, - PipelineService, - MlModelService, - ], + service: ServiceConnectionType, service_type: str, - ) -> Union[ - DashboardService, - DatabaseService, - MessagingService, - PipelineService, - MlModelService, - ]: - service_simple_type = service_type.replace("Service", "").lower() + ) -> ServiceConnectionType: service_connection_type = service.serviceType.value service_name = service.name.__root__ secret_id = self.build_secret_id( - service_simple_type, service_connection_type, service_name + service_type, service_connection_type, service_name ) - connection_py_file = ( - service_connection_type[0].lower() + service_connection_type[1:] + connection_class = self.get_connection_class( + service_type, service_connection_type ) - conn_class = locate( - f"metadata.generated.schema.entity.services.connections.{service_simple_type}.{connection_py_file}Connection.{service_connection_type}Connection" - ) - service.connection.config = conn_class.parse_obj( - json.loads(self._get_string_value(secret_id)) + service_conn_class = self.get_service_connection_class(service_type) + service.connection = service_conn_class( + config=connection_class.parse_obj( + json.loads(self._get_string_value(secret_id)) + ) ) + return service def __init__(self, credentials: AWSCredentials): diff --git a/ingestion/tests/unit/metadata/utils/test_secrets_manager.py b/ingestion/tests/unit/metadata/utils/test_secrets_manager.py index 98f366fe361..81fa497c94a 100644 --- a/ingestion/tests/unit/metadata/utils/test_secrets_manager.py +++ b/ingestion/tests/unit/metadata/utils/test_secrets_manager.py @@ -82,6 +82,8 @@ class TestSecretsManager(TestCase): expected_service = deepcopy(self.service) expected_service.connection.config = self.database_connection + self.service.connection = None + aws_manager.add_service_config_connection(self.service, self.service_type) self.assertEqual(expected_service, self.service) diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/dashboardService.json b/openmetadata-core/src/main/resources/json/schema/entity/services/dashboardService.json index 9f8f3145f9f..42de90f5df0 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/dashboardService.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/dashboardService.json @@ -140,6 +140,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false } diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/databaseService.json b/openmetadata-core/src/main/resources/json/schema/entity/services/databaseService.json index bdbd7f750c2..6bcf79cba21 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/databaseService.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/databaseService.json @@ -279,6 +279,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false } diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/messagingService.json b/openmetadata-core/src/main/resources/json/schema/entity/services/messagingService.json index 055e2a6545b..802decdf141 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/messagingService.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/messagingService.json @@ -110,6 +110,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false } diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/metadataService.json b/openmetadata-core/src/main/resources/json/schema/entity/services/metadataService.json index dfff3a257c6..1a6e034b549 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/metadataService.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/metadataService.json @@ -104,6 +104,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false } diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/mlmodelService.json b/openmetadata-core/src/main/resources/json/schema/entity/services/mlmodelService.json index 11c0b139e8c..017cd202af1 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/mlmodelService.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/mlmodelService.json @@ -102,6 +102,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false } diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/pipelineService.json b/openmetadata-core/src/main/resources/json/schema/entity/services/pipelineService.json index 28e97cf7c71..d2c44bb800f 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/pipelineService.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/pipelineService.json @@ -114,6 +114,6 @@ "default": false } }, - "required": ["id", "name", "serviceType", "connection"], + "required": ["id", "name", "serviceType"], "additionalProperties": false }