From 65517beb62c4bf8715e91f94090aa682d00dae9e Mon Sep 17 00:00:00 2001 From: Nahuel Date: Tue, 9 Aug 2022 16:11:57 +0200 Subject: [PATCH] Fix#5919: Implementation of migration process when a Secrets Manager is configured (#6584) * Implementation of migration process when a Secrets Manager is configured * Update conf/openmetadata.yaml * Address PR comments * Minor change in SQL statment in CollectionDAO * Address Sonar security hotspot issue --- .../catalog/CatalogApplication.java | 5 + .../openmetadata/catalog/EnumInterface.java | 19 ++ .../catalog/ServiceEntityInterface.java | 2 + .../CatalogGenericExceptionMapper.java | 1 - .../exception/SecretsManagerException.java | 13 + .../SecretsManagerMigrationException.java | 25 ++ .../catalog/jdbi3/ChangeEventRepository.java | 17 +- .../catalog/jdbi3/CollectionDAO.java | 3 + .../jdbi3/DashboardServiceRepository.java | 7 +- .../jdbi3/DatabaseServiceRepository.java | 7 +- .../openmetadata/catalog/jdbi3/EntityDAO.java | 4 + .../jdbi3/MessagingServiceRepository.java | 7 +- .../jdbi3/MlModelServiceRepository.java | 7 +- .../jdbi3/PipelineServiceRepository.java | 7 +- ...tory.java => ServiceEntityRepository.java} | 48 ++- .../catalog/resources/CollectionRegistry.java | 10 +- .../resources/events/EventResource.java | 4 +- .../services/ServiceEntityResource.java | 23 +- .../IngestionPipelineResource.java | 4 + .../catalog/secrets/AWSSecretsManager.java | 57 ++-- .../catalog/secrets/LocalSecretsManager.java | 20 +- .../catalog/secrets/SecretsManager.java | 17 +- .../secrets/SecretsManagerConfiguration.java | 13 + .../secrets/SecretsManagerFactory.java | 13 + .../SecretsManagerMigrationService.java | 307 ++++++++++++++++++ .../entity/services/dashboardService.json | 1 + .../entity/services/databaseService.json | 1 + .../entity/services/messagingService.json | 1 + .../entity/services/mlmodelService.json | 1 + .../entity/services/pipelineService.json | 1 + .../DashboardServiceRepositoryUnitTest.java | 15 +- .../DatabaseServiceRepositoryUnitTest.java | 15 +- .../MessagingServiceRepositoryUnitTest.java | 15 +- .../MlModelServiceRepositoryUnitTest.java | 15 +- .../PipelineServiceRepositoryUnitTest.java | 15 +- ....java => ServiceEntityRepositoryTest.java} | 21 +- .../services/ServiceResourceTest.java | 17 +- .../secrets/AWSSecretsManagerTest.java | 12 + .../secrets/LocalSecretsManagerTest.java | 12 + .../secrets/SecretsManagerFactoryTest.java | 12 + conf/openmetadata.yaml | 1 + .../core/entity/interfaces/EnumInterface.java | 19 ++ .../entity/services/dashboardService.json | 3 + .../entity/services/databaseService.json | 3 + .../entity/services/messagingService.json | 3 + .../entity/services/mlmodelService.json | 3 + .../entity/services/pipelineService.json | 3 + 47 files changed, 738 insertions(+), 91 deletions(-) create mode 100644 catalog-rest-service/src/main/java/org/openmetadata/catalog/EnumInterface.java create mode 100644 catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/SecretsManagerMigrationException.java rename catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/{ServiceRepository.java => ServiceEntityRepository.java} (77%) create mode 100644 catalog-rest-service/src/main/java/org/openmetadata/catalog/secrets/SecretsManagerMigrationService.java rename catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/{ServiceRepositoryTest.java => ServiceEntityRepositoryTest.java} (79%) create mode 100644 openmetadata-core/src/main/java/org/openmetadata/core/entity/interfaces/EnumInterface.java diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java index cc5a4ee55f8..95ae309d00c 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java @@ -73,6 +73,7 @@ import org.openmetadata.catalog.resources.CollectionRegistry; import org.openmetadata.catalog.resources.search.SearchResource; import org.openmetadata.catalog.secrets.SecretsManager; import org.openmetadata.catalog.secrets.SecretsManagerFactory; +import org.openmetadata.catalog.secrets.SecretsManagerMigrationService; import org.openmetadata.catalog.security.AuthenticationConfiguration; import org.openmetadata.catalog.security.Authorizer; import org.openmetadata.catalog.security.AuthorizerConfiguration; @@ -158,6 +159,10 @@ public class CatalogApplication extends Application { // Register Event publishers registerEventPublisher(catalogConfig); + // Check if migration is need from local secret manager to configured one and migrate + new SecretsManagerMigrationService(secretsManager, catalogConfig.getClusterName()) + .migrateServicesToSecretManagerIfNeeded(); + // start authorizer after event publishers // authorizer creates admin/bot users, ES publisher should start before to index users created by authorizer authorizer.init(catalogConfig.getAuthorizerConfiguration(), jdbi); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/EnumInterface.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/EnumInterface.java new file mode 100644 index 00000000000..4f51cafce50 --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/EnumInterface.java @@ -0,0 +1,19 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog; + +/** Interface which could be implemented by Enums classes */ +public interface EnumInterface { + String value(); +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/ServiceEntityInterface.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/ServiceEntityInterface.java index af1be465508..fe6d8ff2413 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/ServiceEntityInterface.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/ServiceEntityInterface.java @@ -24,4 +24,6 @@ public interface ServiceEntityInterface extends EntityInterface { ServiceEntityInterface withOwner(EntityReference owner); void setPipelines(List pipelines); + + EnumInterface getServiceType(); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogGenericExceptionMapper.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogGenericExceptionMapper.java index f6c7c4fed4f..9652dfe6c9b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogGenericExceptionMapper.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogGenericExceptionMapper.java @@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory; public class CatalogGenericExceptionMapper implements ExceptionMapper { @Override public Response toResponse(Throwable ex) { - ex.printStackTrace(); LOG.debug(ex.getMessage()); if (ex instanceof ProcessingException || ex instanceof IllegalArgumentException) { final Response response = BadRequestException.of().getResponse(); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/SecretsManagerException.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/SecretsManagerException.java index 6b6bf455549..97e9abe6c71 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/SecretsManagerException.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/SecretsManagerException.java @@ -1,3 +1,16 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.openmetadata.catalog.exception; import javax.ws.rs.core.Response; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/SecretsManagerMigrationException.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/SecretsManagerMigrationException.java new file mode 100644 index 00000000000..1b04af8d1d7 --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/SecretsManagerMigrationException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.catalog.exception; + +public class SecretsManagerMigrationException extends RuntimeException { + + public SecretsManagerMigrationException(String message, Throwable throwable) { + super(message, throwable); + } + + public SecretsManagerMigrationException(String message) { + super(message); + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java index 2f3f24041fa..924d9859a0a 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java @@ -26,10 +26,10 @@ import org.openmetadata.catalog.type.ChangeEvent; import org.openmetadata.catalog.util.JsonUtils; public class ChangeEventRepository { - private final CollectionDAO dao; + private final CollectionDAO.ChangeEventDAO dao; public ChangeEventRepository(CollectionDAO dao) { - this.dao = dao; + this.dao = dao.changeEventDAO(); } @Transaction @@ -37,10 +37,10 @@ public class ChangeEventRepository { long timestamp, List entityCreatedList, List entityUpdatedList, List entityDeletedList) throws IOException { List jsons = new ArrayList<>(); - jsons.addAll(dao.changeEventDAO().list(ENTITY_CREATED.value(), entityCreatedList, timestamp)); - jsons.addAll(dao.changeEventDAO().list(ENTITY_UPDATED.value(), entityUpdatedList, timestamp)); - jsons.addAll(dao.changeEventDAO().list(ENTITY_DELETED.value(), entityDeletedList, timestamp)); - jsons.addAll(dao.changeEventDAO().list(ENTITY_SOFT_DELETED.value(), entityDeletedList, timestamp)); + jsons.addAll(dao.list(ENTITY_CREATED.value(), entityCreatedList, timestamp)); + jsons.addAll(dao.list(ENTITY_UPDATED.value(), entityUpdatedList, timestamp)); + jsons.addAll(dao.list(ENTITY_DELETED.value(), entityDeletedList, timestamp)); + jsons.addAll(dao.list(ENTITY_SOFT_DELETED.value(), entityDeletedList, timestamp)); List changeEvents = new ArrayList<>(); for (String json : jsons) { @@ -48,4 +48,9 @@ public class ChangeEventRepository { } return changeEvents; } + + @Transaction + public void deleteAll(String entityType) throws IOException { + dao.deleteAll(entityType); + } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java index 3e53f1e07ac..943a73389e1 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java @@ -2390,6 +2390,9 @@ public interface CollectionDAO { connectionType = POSTGRES) void insert(@Bind("json") String json); + @SqlUpdate("DELETE FROM change_event WHERE entityType = :entityType") + void deleteAll(@Bind("entityType") String entityType); + default List list(String eventType, List entityTypes, long timestamp) { if (CommonUtil.nullOrEmpty(entityTypes)) { return Collections.emptyList(); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java index 1a0d52c4136..14bbf3209c3 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java @@ -20,7 +20,7 @@ import org.openmetadata.catalog.resources.services.dashboard.DashboardServiceRes import org.openmetadata.catalog.secrets.SecretsManager; import org.openmetadata.catalog.type.DashboardConnection; -public class DashboardServiceRepository extends ServiceRepository { +public class DashboardServiceRepository extends ServiceEntityRepository { public DashboardServiceRepository(CollectionDAO dao, SecretsManager secretsManager) { super( @@ -32,9 +32,4 @@ public class DashboardServiceRepository extends ServiceRepository { +public class DatabaseServiceRepository extends ServiceEntityRepository { public DatabaseServiceRepository(CollectionDAO dao, SecretsManager secretsManager) { super( DatabaseServiceResource.COLLECTION_PATH, @@ -31,9 +31,4 @@ public class DatabaseServiceRepository extends ServiceRepository { update(getTableName(), id.toString(), json); } + default void update(EntityInterface entity) throws JsonProcessingException { + update(getTableName(), entity.getId().toString(), JsonUtils.pojoToJson(entity)); + } + default String getCondition(Include include) { if (!supportsSoftDelete()) { return ""; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java index 903750b8f9c..7dec4888fe7 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java @@ -20,7 +20,7 @@ import org.openmetadata.catalog.resources.services.messaging.MessagingServiceRes import org.openmetadata.catalog.secrets.SecretsManager; import org.openmetadata.catalog.type.MessagingConnection; -public class MessagingServiceRepository extends ServiceRepository { +public class MessagingServiceRepository extends ServiceEntityRepository { private static final String UPDATE_FIELDS = "owner, connection"; public MessagingServiceRepository(CollectionDAO dao, SecretsManager secretsManager) { @@ -34,9 +34,4 @@ public class MessagingServiceRepository extends ServiceRepository { +public class MlModelServiceRepository extends ServiceEntityRepository { private static final String UPDATE_FIELDS = "owner,connection"; public MlModelServiceRepository(CollectionDAO dao, SecretsManager secretsManager) { @@ -34,9 +34,4 @@ public class MlModelServiceRepository extends ServiceRepository { +public class PipelineServiceRepository extends ServiceEntityRepository { public PipelineServiceRepository(CollectionDAO dao, SecretsManager secretsManager) { super( @@ -32,9 +32,4 @@ public class PipelineServiceRepository extends ServiceRepository +public abstract class ServiceEntityRepository< + T extends ServiceEntityInterface, S extends ServiceConnectionEntityInterface> extends EntityRepository { private static final String UPDATE_FIELDS = "owner"; - private final Class serviceConnectionClass; + @Getter private final Class serviceConnectionClass; protected final SecretsManager secretsManager; - private final ServiceType serviceType; + @Getter private final ServiceType serviceType; - protected ServiceRepository( + protected ServiceEntityRepository( String collectionPath, String service, CollectionDAO dao, @@ -37,7 +51,7 @@ public abstract class ServiceRepository getCollectionMap() { + public Map getCollectionMap() { return Collections.unmodifiableMap(collectionMap); } @@ -135,6 +137,7 @@ public final class CollectionRegistry { CollectionDAO daoObject = jdbi.onDemand(CollectionDAO.class); Objects.requireNonNull(daoObject, "CollectionDAO must not be null"); Object resource = createResource(daoObject, resourceClass, config, authorizer, secretsManager); + details.setResource(resource); environment.jersey().register(resource); LOG.info("Registering {} with order {}", resourceClass, details.order); } catch (Exception ex) { @@ -232,10 +235,13 @@ public final class CollectionRegistry { public static class CollectionDetails { private final int order; - private final String resourceClass; + + @Getter private final String resourceClass; private final CollectionDescriptor cd; private final List childCollections = new ArrayList<>(); + @Getter @Setter private Object resource; + CollectionDetails(CollectionDescriptor cd, String resourceClass, int order) { this.cd = cd; this.resourceClass = resourceClass; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java index 39f0ed8cb83..b22382e0193 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java @@ -31,6 +31,7 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.UriInfo; +import lombok.Getter; import org.openmetadata.catalog.Entity.EntityList; import org.openmetadata.catalog.jdbi3.ChangeEventRepository; import org.openmetadata.catalog.jdbi3.CollectionDAO; @@ -46,7 +47,8 @@ import org.openmetadata.catalog.util.ResultList; @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "events") public class EventResource { - private final ChangeEventRepository dao; + + @Getter private final ChangeEventRepository dao; private final Authorizer authorizer; public static class ChangeEventList extends ResultList { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ServiceEntityResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ServiceEntityResource.java index 2c657324f24..fa4648271f6 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ServiceEntityResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ServiceEntityResource.java @@ -1,12 +1,26 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.openmetadata.catalog.resources.services; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import javax.ws.rs.core.SecurityContext; +import lombok.Getter; import org.openmetadata.catalog.ServiceConnectionEntityInterface; import org.openmetadata.catalog.ServiceEntityInterface; import org.openmetadata.catalog.entity.services.ServiceType; -import org.openmetadata.catalog.jdbi3.ServiceRepository; +import org.openmetadata.catalog.jdbi3.ServiceEntityRepository; import org.openmetadata.catalog.resources.EntityResource; import org.openmetadata.catalog.secrets.SecretsManager; import org.openmetadata.catalog.security.AuthorizationException; @@ -14,11 +28,15 @@ import org.openmetadata.catalog.security.Authorizer; import org.openmetadata.catalog.util.ResultList; public abstract class ServiceEntityResource< - T extends ServiceEntityInterface, R extends ServiceRepository, S extends ServiceConnectionEntityInterface> + T extends ServiceEntityInterface, + R extends ServiceEntityRepository, + S extends ServiceConnectionEntityInterface> extends EntityResource { private final SecretsManager secretsManager; + @Getter private final ServiceEntityRepository serviceEntityRepository; + private final ServiceType serviceType; protected ServiceEntityResource( @@ -28,6 +46,7 @@ public abstract class ServiceEntityResource< SecretsManager secretsManager, ServiceType serviceType) { super(entityClass, serviceRepository, authorizer); + this.serviceEntityRepository = serviceRepository; this.secretsManager = secretsManager; this.serviceType = serviceType; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java index e31f763b826..0a8b47fd2bc 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java @@ -51,6 +51,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.openmetadata.catalog.CatalogApplicationConfig; import org.openmetadata.catalog.Entity; @@ -84,6 +85,8 @@ public class IngestionPipelineResource extends EntityResourceIt will migrate all the entities with connection parameters: + * + *

- the connection config objects of services entities which implement the ServiceEntityResource and + * ServiceEntityRepository (services using the secrets' manager) + * + *

- remove the auth security config in the IngestionPipeline entities + * + *

- remove all the ChangeEvent entities related to ingestion pipelines and services + */ +@Slf4j +public class SecretsManagerMigrationService { + private final SecretsManager newSecretManager; + + private final SecretsManager oldSecretManager; + + private final Map, ServiceEntityRepository> + connectionTypeRepositoriesMap; + + private final ChangeEventRepository changeEventRepository; + + private final IngestionPipelineRepository ingestionPipelineRepository; + + public SecretsManagerMigrationService(SecretsManager secretsManager, String clusterName) { + this.newSecretManager = secretsManager; + this.connectionTypeRepositoriesMap = retrieveConnectionTypeRepositoriesMap(); + this.changeEventRepository = retrieveChangeEventRepository(); + this.ingestionPipelineRepository = retrieveIngestionPipelineRepository(); + // by default, it is going to be LOCAL + this.oldSecretManager = SecretsManagerFactory.createSecretsManager(null, clusterName); + } + + public void migrateServicesToSecretManagerIfNeeded() { + if (!newSecretManager.isLocal()) { + migrateServices(); + migrateIngestionPipelines(); + } else { + LOG.info("Local secrets manager does not need to check if migration is needed."); + } + } + + private void migrateServices() { + LOG.info( + String.format( + "Checking if services migration is needed for secrets manager: [{}]", + newSecretManager.getSecretsManagerProvider())); + List notStoredServices = retrieveNotStoredServices(); + if (!notStoredServices.isEmpty()) { + notStoredServices.forEach(this::migrateService); + deleteChangeEventsForServices(); + } else { + LOG.info( + String.format( + "All services are already safely stored in [{}] secrets manager", + newSecretManager.getSecretsManagerProvider())); + } + } + + private void migrateIngestionPipelines() { + LOG.info( + String.format( + "Checking if ingestion pipelines migration is needed for secrets manager: [{}]", + newSecretManager.getSecretsManagerProvider())); + List notStoredIngestionPipelines = retrieveNotStoredIngestionPipelines(); + if (!notStoredIngestionPipelines.isEmpty()) { + notStoredIngestionPipelines.forEach(this::migrateIngestionPipelines); + deleteChangeEventsForIngestionPipelines(); + } else { + LOG.info( + String.format( + "All ingestion pipelines are already safely stored in [{}] secrets manager", + newSecretManager.getSecretsManagerProvider())); + } + } + + private void migrateService(ServiceEntityInterface serviceEntityInterface) { + ServiceEntityRepository repository = + connectionTypeRepositoriesMap.get(serviceEntityInterface.getConnection().getClass()); + try { + ServiceEntityInterface service = repository.dao.findEntityById(serviceEntityInterface.getId()); + // we have to decrypt using the old secrets manager and encrypt again with the new one + service + .getConnection() + .setConfig( + oldSecretManager.encryptOrDecryptServiceConnectionConfig( + service.getConnection().getConfig(), + service.getServiceType().value(), + service.getName(), + repository.getServiceType(), + false)); + newSecretManager.encryptOrDecryptServiceConnectionConfig( + service.getConnection().getConfig(), + service.getServiceType().value(), + service.getName(), + repository.getServiceType(), + true); + // avoid reaching secrets manager quotas + Thread.sleep(100); + service.getConnection().setConfig(null); + repository.dao.update(service); + } catch (IOException | InterruptedException e) { + throw new SecretsManagerMigrationException(e.getMessage(), e.getCause()); + } + } + + private List retrieveNotStoredServices() { + return connectionTypeRepositoriesMap.values().stream() + .map(this::retrieveServices) + .flatMap(List::stream) + .collect(Collectors.toList()); + } + + private List retrieveServices(ServiceEntityRepository serviceEntityRepository) { + try { + return serviceEntityRepository + .listAfter( + null, + EntityUtil.Fields.EMPTY_FIELDS, + new ListFilter(), + serviceEntityRepository.dao.listCount(new ListFilter()), + null) + .getData().stream() + .map(ServiceEntityInterface.class::cast) + .filter( + service -> + !Objects.isNull(service.getConnection()) && !Objects.isNull(service.getConnection().getConfig())) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new SecretsManagerMigrationException(e.getMessage(), e.getCause()); + } + } + + private void migrateIngestionPipelines(IngestionPipeline ingestionPipeline) { + try { + IngestionPipeline ingestion = ingestionPipelineRepository.dao.findEntityById(ingestionPipeline.getId()); + ingestion.getOpenMetadataServerConnection().setSecurityConfig(null); + ingestionPipelineRepository.dao.update(ingestion); + } catch (IOException e) { + throw new SecretsManagerMigrationException(e.getMessage(), e.getCause()); + } + } + + private List retrieveNotStoredIngestionPipelines() { + try { + return ingestionPipelineRepository + .listAfter( + null, + EntityUtil.Fields.EMPTY_FIELDS, + new ListFilter(), + ingestionPipelineRepository.dao.listCount(new ListFilter()), + null) + .getData().stream() + .filter( + ingestionPipeline -> + !Objects.isNull(ingestionPipeline.getOpenMetadataServerConnection()) + && !Objects.isNull(ingestionPipeline.getOpenMetadataServerConnection().getSecurityConfig())) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new SecretsManagerMigrationException(e.getMessage(), e.getCause()); + } + } + + /** This method delete all the change events which could contain connection config parameters for services */ + private void deleteChangeEventsForServices() { + connectionTypeRepositoriesMap.values().stream() + .map(ServiceEntityRepository::getServiceType) + .forEach( + serviceType -> { + try { + changeEventRepository.deleteAll( + Entity.class + .getField(serviceType.value().toUpperCase(Locale.ROOT) + "_SERVICE") + .get(Entity.class) + .toString()); + } catch (NoSuchFieldException | IOException | IllegalAccessException e) { + throw new SecretsManagerMigrationException(e.getMessage(), e.getCause()); + } + }); + } + + /** + * This method delete all the change events which could contain auth provider config parameters for ingestion + * pipelines + */ + private void deleteChangeEventsForIngestionPipelines() { + ChangeEventRepository changeEventRepository = retrieveChangeEventRepository(); + try { + changeEventRepository.deleteAll(Entity.INGESTION_PIPELINE); + } catch (IOException e) { + throw new SecretsManagerMigrationException(e.getMessage(), e.getCause()); + } + } + + private Map, ServiceEntityRepository> + retrieveConnectionTypeRepositoriesMap() { + Map, ServiceEntityRepository> + connectionTypeRepositoriesMap = + CollectionRegistry.getInstance().getCollectionMap().values().stream() + .map(this::retrieveServiceRepository) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toMap(ServiceEntityRepository::getServiceConnectionClass, Function.identity())); + if (connectionTypeRepositoriesMap.isEmpty()) { + throw new SecretsManagerMigrationException("Unexpected error: ServiceRepository not found."); + } + return connectionTypeRepositoriesMap; + } + + private ChangeEventRepository retrieveChangeEventRepository() { + return CollectionRegistry.getInstance().getCollectionMap().values().stream() + .map(this::retrieveChangeEventRepository) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst() + .orElseThrow(() -> new SecretsManagerMigrationException("Unexpected error: ChangeEventRepository not found.")); + } + + private IngestionPipelineRepository retrieveIngestionPipelineRepository() { + return CollectionRegistry.getInstance().getCollectionMap().values().stream() + .map(this::retrieveIngestionPipelineRepository) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst() + .orElseThrow( + () -> new SecretsManagerMigrationException("Unexpected error: IngestionPipelineRepository not found.")); + } + + private Optional retrieveIngestionPipelineRepository( + CollectionRegistry.CollectionDetails collectionDetails) { + Class collectionDetailsClass = extractCollectionDetailsClass(collectionDetails); + if (IngestionPipelineResource.class.equals(collectionDetailsClass)) { + return Optional.of( + ((IngestionPipelineResource) collectionDetails.getResource()).getIngestionPipelineRepository()); + } + return Optional.empty(); + } + + private Optional retrieveChangeEventRepository( + CollectionRegistry.CollectionDetails collectionDetails) { + Class collectionDetailsClass = extractCollectionDetailsClass(collectionDetails); + if (EventResource.class.equals(collectionDetailsClass)) { + return Optional.of(((EventResource) collectionDetails.getResource()).getDao()); + } + return Optional.empty(); + } + + private Optional> retrieveServiceRepository( + CollectionRegistry.CollectionDetails collectionDetails) { + Class collectionDetailsClass = extractCollectionDetailsClass(collectionDetails); + if (ServiceEntityResource.class.isAssignableFrom(collectionDetailsClass)) { + return Optional.of( + ((ServiceEntityResource) collectionDetails.getResource()).getServiceEntityRepository()); + } + return Optional.empty(); + } + + private Class extractCollectionDetailsClass(CollectionRegistry.CollectionDetails collectionDetails) { + Class collectionDetailsClass; + try { + collectionDetailsClass = Class.forName(collectionDetails.getResourceClass()); + } catch (ClassNotFoundException e) { + throw new SecretsManagerMigrationException(e.getMessage(), e.getCause()); + } + return collectionDetailsClass; + } +} diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json index 713bfa75953..1819649233a 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json @@ -13,6 +13,7 @@ "dashboardServiceType": { "description": "Type of Dashboard service - Superset, Looker, Redash, Tableau, Metabase, PowerBi or Mode", "type": "string", + "javaInterfaces": ["org.openmetadata.catalog.EnumInterface"], "enum": [ "Superset", "Looker", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json index 60c4d44241c..b2651bc47f8 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json @@ -12,6 +12,7 @@ "definitions": { "databaseServiceType": { "description": "Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres...", + "javaInterfaces": ["org.openmetadata.catalog.EnumInterface"], "type": "string", "enum": [ "BigQuery", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json index aee7f294a8b..81a3cbbff7c 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json @@ -13,6 +13,7 @@ "messagingServiceType": { "description": "Type of messaging service - Kafka or Pulsar.", "type": "string", + "javaInterfaces": ["org.openmetadata.catalog.EnumInterface"], "enum": ["Kafka", "Pulsar"], "javaEnums": [ { diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/mlmodelService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/mlmodelService.json index f4f48f5ce18..7c90dfec7aa 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/mlmodelService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/mlmodelService.json @@ -13,6 +13,7 @@ "mlModelServiceType": { "description": "Type of MlModel service", "type": "string", + "javaInterfaces": ["org.openmetadata.catalog.EnumInterface"], "enum": ["Mlflow", "Sklearn"], "javaEnums": [ { diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json index ea034b074c2..3549366de07 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json @@ -13,6 +13,7 @@ "pipelineServiceType": { "description": "Type of pipeline service - Airflow or Prefect.", "type": "string", + "javaInterfaces": ["org.openmetadata.catalog.EnumInterface"], "enum": ["Airflow", "Glue", "Airbyte", "Fivetran"], "javaEnums": [ { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepositoryUnitTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepositoryUnitTest.java index a34b3941048..4e065fa1f1c 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepositoryUnitTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepositoryUnitTest.java @@ -1,3 +1,16 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.openmetadata.catalog.jdbi3; import static org.mockito.ArgumentMatchers.isNull; @@ -12,7 +25,7 @@ import org.openmetadata.catalog.services.connections.database.MysqlConnection; import org.openmetadata.catalog.type.DashboardConnection; public class DashboardServiceRepositoryUnitTest - extends ServiceRepositoryTest { + extends ServiceEntityRepositoryTest { protected DashboardServiceRepositoryUnitTest() { super(ServiceType.DASHBOARD); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepositoryUnitTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepositoryUnitTest.java index 2c35032a70b..cf7404efbb6 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepositoryUnitTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepositoryUnitTest.java @@ -1,3 +1,16 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.openmetadata.catalog.jdbi3; import static org.mockito.ArgumentMatchers.isNull; @@ -12,7 +25,7 @@ import org.openmetadata.catalog.secrets.SecretsManager; import org.openmetadata.catalog.services.connections.database.MysqlConnection; public class DatabaseServiceRepositoryUnitTest - extends ServiceRepositoryTest { + extends ServiceEntityRepositoryTest { protected DatabaseServiceRepositoryUnitTest() { super(ServiceType.DATABASE); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepositoryUnitTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepositoryUnitTest.java index 45f70a28472..adaa7b7e97e 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepositoryUnitTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepositoryUnitTest.java @@ -1,3 +1,16 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.openmetadata.catalog.jdbi3; import static org.mockito.ArgumentMatchers.isNull; @@ -12,7 +25,7 @@ import org.openmetadata.catalog.services.connections.database.MysqlConnection; import org.openmetadata.catalog.type.MessagingConnection; public class MessagingServiceRepositoryUnitTest - extends ServiceRepositoryTest { + extends ServiceEntityRepositoryTest { protected MessagingServiceRepositoryUnitTest() { super(ServiceType.MESSAGING); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/MlModelServiceRepositoryUnitTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/MlModelServiceRepositoryUnitTest.java index 650c5fbb6b4..7ea41a6636a 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/MlModelServiceRepositoryUnitTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/MlModelServiceRepositoryUnitTest.java @@ -1,3 +1,16 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.openmetadata.catalog.jdbi3; import static org.mockito.ArgumentMatchers.isNull; @@ -12,7 +25,7 @@ import org.openmetadata.catalog.services.connections.database.MysqlConnection; import org.openmetadata.catalog.type.MlModelConnection; public class MlModelServiceRepositoryUnitTest - extends ServiceRepositoryTest { + extends ServiceEntityRepositoryTest { protected MlModelServiceRepositoryUnitTest() { super(ServiceType.ML_MODEL); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepositoryUnitTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepositoryUnitTest.java index af78c3d435a..b691d17bb0e 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepositoryUnitTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepositoryUnitTest.java @@ -1,3 +1,16 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.openmetadata.catalog.jdbi3; import static org.mockito.ArgumentMatchers.isNull; @@ -12,7 +25,7 @@ import org.openmetadata.catalog.services.connections.database.MysqlConnection; import org.openmetadata.catalog.type.PipelineConnection; public class PipelineServiceRepositoryUnitTest - extends ServiceRepositoryTest { + extends ServiceEntityRepositoryTest { protected PipelineServiceRepositoryUnitTest() { super(ServiceType.PIPELINE); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/ServiceRepositoryTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/ServiceEntityRepositoryTest.java similarity index 79% rename from catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/ServiceRepositoryTest.java rename to catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/ServiceEntityRepositoryTest.java index 6a1bb0ca3cf..2c0ba26b8bc 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/ServiceRepositoryTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/ServiceEntityRepositoryTest.java @@ -1,3 +1,16 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.openmetadata.catalog.jdbi3; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -23,8 +36,10 @@ import org.openmetadata.catalog.entity.services.ServiceType; import org.openmetadata.catalog.secrets.SecretsManager; @ExtendWith(MockitoExtension.class) -public abstract class ServiceRepositoryTest< - T extends ServiceRepository, R extends ServiceEntityInterface, S extends ServiceConnectionEntityInterface> { +public abstract class ServiceEntityRepositoryTest< + T extends ServiceEntityRepository, + R extends ServiceEntityInterface, + S extends ServiceConnectionEntityInterface> { @Mock protected CollectionDAO collectionDAO; @@ -38,7 +53,7 @@ public abstract class ServiceRepositoryTest< private final ServiceType expectedServiceType; - protected ServiceRepositoryTest(ServiceType serviceType) { + protected ServiceEntityRepositoryTest(ServiceType serviceType) { this.expectedServiceType = serviceType; } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ServiceResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ServiceResourceTest.java index 97934e6e77e..1e22d93e1ec 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ServiceResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ServiceResourceTest.java @@ -1,3 +1,16 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.openmetadata.catalog.resources.services; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -25,7 +38,7 @@ import org.openmetadata.catalog.ServiceConnectionEntityInterface; import org.openmetadata.catalog.ServiceEntityInterface; import org.openmetadata.catalog.entity.services.ServiceType; import org.openmetadata.catalog.jdbi3.CollectionDAO; -import org.openmetadata.catalog.jdbi3.ServiceRepository; +import org.openmetadata.catalog.jdbi3.ServiceEntityRepository; import org.openmetadata.catalog.secrets.SecretsManager; import org.openmetadata.catalog.security.AuthorizationException; import org.openmetadata.catalog.security.Authorizer; @@ -34,7 +47,7 @@ import org.openmetadata.catalog.security.Authorizer; public abstract class ServiceResourceTest< T extends ServiceEntityResource, R extends ServiceEntityInterface, - S extends ServiceRepository, + S extends ServiceEntityRepository, U extends ServiceConnectionEntityInterface> { T serviceResource; diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/AWSSecretsManagerTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/AWSSecretsManagerTest.java index 8de3e550883..b6648a32c07 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/AWSSecretsManagerTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/AWSSecretsManagerTest.java @@ -1,3 +1,15 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.openmetadata.catalog.secrets; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/LocalSecretsManagerTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/LocalSecretsManagerTest.java index 334c66b006f..89c295278aa 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/LocalSecretsManagerTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/LocalSecretsManagerTest.java @@ -1,3 +1,15 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.openmetadata.catalog.secrets; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/SecretsManagerFactoryTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/SecretsManagerFactoryTest.java index 53d8fac5fd2..3252639e485 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/SecretsManagerFactoryTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/SecretsManagerFactoryTest.java @@ -1,3 +1,15 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.openmetadata.catalog.secrets; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index e279a75c208..f82bc1a424c 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -228,6 +228,7 @@ fernetConfiguration: secretsManagerConfiguration: secretsManager: ${SECRET_MANAGER:-local} # Possible values are "local", "aws" # secretsManager: aws +# it will use the default auth provider for the secrets' manager service if parameters are not set # parameters: # region: # accessKeyId: diff --git a/openmetadata-core/src/main/java/org/openmetadata/core/entity/interfaces/EnumInterface.java b/openmetadata-core/src/main/java/org/openmetadata/core/entity/interfaces/EnumInterface.java new file mode 100644 index 00000000000..4cd325cdbdb --- /dev/null +++ b/openmetadata-core/src/main/java/org/openmetadata/core/entity/interfaces/EnumInterface.java @@ -0,0 +1,19 @@ +/* + * Copyright 2022 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.core.entity.interfaces; + +/** Interface which could be implemented by Enums classes */ +public interface EnumInterface { + String value(); +} diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/dashboardService.json b/openmetadata-core/src/main/resources/json/schema/entity/services/dashboardService.json index 42de90f5df0..fb12701759c 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/dashboardService.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/dashboardService.json @@ -10,6 +10,9 @@ "dashboardServiceType": { "description": "Type of Dashboard service - Superset, Looker, Redash, Tableau, Metabase, PowerBi or Mode", "type": "string", + "javaInterfaces": [ + "org.openmetadata.core.entity.interfaces.EnumInterface" + ], "enum": [ "Superset", "Looker", diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/databaseService.json b/openmetadata-core/src/main/resources/json/schema/entity/services/databaseService.json index 43e651146fd..c14af7339d9 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/databaseService.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/databaseService.json @@ -10,6 +10,9 @@ "databaseServiceType": { "description": "Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres...", "type": "string", + "javaInterfaces": [ + "org.openmetadata.core.entity.interfaces.EnumInterface" + ], "enum": [ "BigQuery", "Mysql", diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/messagingService.json b/openmetadata-core/src/main/resources/json/schema/entity/services/messagingService.json index 802decdf141..179d073b56a 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/messagingService.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/messagingService.json @@ -10,6 +10,9 @@ "messagingServiceType": { "description": "Type of messaging service - Kafka or Pulsar.", "type": "string", + "javaInterfaces": [ + "org.openmetadata.core.entity.interfaces.EnumInterface" + ], "enum": ["Kafka", "Pulsar"], "javaEnums": [ { diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/mlmodelService.json b/openmetadata-core/src/main/resources/json/schema/entity/services/mlmodelService.json index 017cd202af1..59445a1fb7c 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/mlmodelService.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/mlmodelService.json @@ -10,6 +10,9 @@ "mlModelServiceType": { "description": "Type of MlModel service", "type": "string", + "javaInterfaces": [ + "org.openmetadata.core.entity.interfaces.EnumInterface" + ], "enum": ["Mlflow", "Sklearn"], "javaEnums": [ { diff --git a/openmetadata-core/src/main/resources/json/schema/entity/services/pipelineService.json b/openmetadata-core/src/main/resources/json/schema/entity/services/pipelineService.json index d2c44bb800f..c14be0ebc90 100644 --- a/openmetadata-core/src/main/resources/json/schema/entity/services/pipelineService.json +++ b/openmetadata-core/src/main/resources/json/schema/entity/services/pipelineService.json @@ -10,6 +10,9 @@ "pipelineServiceType": { "description": "Type of pipeline service - Airflow or Prefect.", "type": "string", + "javaInterfaces": [ + "org.openmetadata.core.entity.interfaces.EnumInterface" + ], "enum": ["Airflow", "Prefect", "Glue", "Generic", "Airbyte"], "javaEnums": [ {