yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-13 00:22:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix#5919: Implementation of migration process when a Secrets Manager is configured (#6584)

Browse Source 
* Implementation of migration process when a Secrets Manager is configured

* Update conf/openmetadata.yaml

* Address PR comments

* Minor change in SQL statment in CollectionDAO

* Address Sonar security hotspot issue
This commit is contained in:
Nahuel 2022-08-09 16:11:57 +02:00 committed by GitHub
parent e4aa443b65
commit 65517beb62
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
47 changed files with 738 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java
View File

@ -73,6 +73,7 @@ import org.openmetadata.catalog.resources.CollectionRegistry;
import org.openmetadata.catalog.resources.search.SearchResource;
import org.openmetadata.catalog.secrets.SecretsManager;
import org.openmetadata.catalog.secrets.SecretsManagerFactory;
import org.openmetadata.catalog.secrets.SecretsManagerMigrationService;
import org.openmetadata.catalog.security.AuthenticationConfiguration;
import org.openmetadata.catalog.security.Authorizer;
import org.openmetadata.catalog.security.AuthorizerConfiguration;
@ -158,6 +159,10 @@ public class CatalogApplication extends Application<CatalogApplicationConfig> {
// Register Event publishers
registerEventPublisher(catalogConfig);
// Check if migration is need from local secret manager to configured one and migrate
new SecretsManagerMigrationService(secretsManager, catalogConfig.getClusterName())
.migrateServicesToSecretManagerIfNeeded();
// start authorizer after event publishers
// authorizer creates admin/bot users, ES publisher should start before to index users created by authorizer
authorizer.init(catalogConfig.getAuthorizerConfiguration(), jdbi);

19
catalog-rest-service/src/main/java/org/openmetadata/catalog/EnumInterface.java Normal file
View File

@ -0,0 +1,19 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog;
/** Interface which could be implemented by Enums classes */
public interface EnumInterface {
String value();
}

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/ServiceEntityInterface.java
View File

@ -24,4 +24,6 @@ public interface ServiceEntityInterface extends EntityInterface {
ServiceEntityInterface withOwner(EntityReference owner);
void setPipelines(List<EntityReference> pipelines);
EnumInterface getServiceType();
}

1
catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/CatalogGenericExceptionMapper.java
View File

@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory;
public class CatalogGenericExceptionMapper implements ExceptionMapper<Throwable> {
@Override
public Response toResponse(Throwable ex) {
ex.printStackTrace();
LOG.debug(ex.getMessage());
if (ex instanceof ProcessingException || ex instanceof IllegalArgumentException) {
final Response response = BadRequestException.of().getResponse();

13
catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/SecretsManagerException.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.exception;
import javax.ws.rs.core.Response;

25
catalog-rest-service/src/main/java/org/openmetadata/catalog/exception/SecretsManagerMigrationException.java Normal file
View File

@ -0,0 +1,25 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.exception;
public class SecretsManagerMigrationException extends RuntimeException {
public SecretsManagerMigrationException(String message, Throwable throwable) {
super(message, throwable);
}
public SecretsManagerMigrationException(String message) {
super(message);
}
}

17
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChangeEventRepository.java
View File

@ -26,10 +26,10 @@ import org.openmetadata.catalog.type.ChangeEvent;
import org.openmetadata.catalog.util.JsonUtils;
public class ChangeEventRepository {
private final CollectionDAO dao;
private final CollectionDAO.ChangeEventDAO dao;
public ChangeEventRepository(CollectionDAO dao) {
this.dao = dao;
this.dao = dao.changeEventDAO();
}
@Transaction
@ -37,10 +37,10 @@ public class ChangeEventRepository {
long timestamp, List<String> entityCreatedList, List<String> entityUpdatedList, List<String> entityDeletedList)
throws IOException {
List<String> jsons = new ArrayList<>();
jsons.addAll(dao.changeEventDAO().list(ENTITY_CREATED.value(), entityCreatedList, timestamp));
jsons.addAll(dao.changeEventDAO().list(ENTITY_UPDATED.value(), entityUpdatedList, timestamp));
jsons.addAll(dao.changeEventDAO().list(ENTITY_DELETED.value(), entityDeletedList, timestamp));
jsons.addAll(dao.changeEventDAO().list(ENTITY_SOFT_DELETED.value(), entityDeletedList, timestamp));
jsons.addAll(dao.list(ENTITY_CREATED.value(), entityCreatedList, timestamp));
jsons.addAll(dao.list(ENTITY_UPDATED.value(), entityUpdatedList, timestamp));
jsons.addAll(dao.list(ENTITY_DELETED.value(), entityDeletedList, timestamp));
jsons.addAll(dao.list(ENTITY_SOFT_DELETED.value(), entityDeletedList, timestamp));
List<ChangeEvent> changeEvents = new ArrayList<>();
for (String json : jsons) {
@ -48,4 +48,9 @@ public class ChangeEventRepository {
}
return changeEvents;
}
@Transaction
public void deleteAll(String entityType) throws IOException {
dao.deleteAll(entityType);
}
}

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/CollectionDAO.java
View File

@ -2390,6 +2390,9 @@ public interface CollectionDAO {
connectionType = POSTGRES)
void insert(@Bind("json") String json);
@SqlUpdate("DELETE FROM change_event WHERE entityType = :entityType")
void deleteAll(@Bind("entityType") String entityType);
default List<String> list(String eventType, List<String> entityTypes, long timestamp) {
if (CommonUtil.nullOrEmpty(entityTypes)) {
return Collections.emptyList();

7
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java
View File

@ -20,7 +20,7 @@ import org.openmetadata.catalog.resources.services.dashboard.DashboardServiceRes
import org.openmetadata.catalog.secrets.SecretsManager;
import org.openmetadata.catalog.type.DashboardConnection;
public class DashboardServiceRepository extends ServiceRepository<DashboardService, DashboardConnection> {
public class DashboardServiceRepository extends ServiceEntityRepository<DashboardService, DashboardConnection> {
public DashboardServiceRepository(CollectionDAO dao, SecretsManager secretsManager) {
super(
@ -32,9 +32,4 @@ public class DashboardServiceRepository extends ServiceRepository<DashboardServi
DashboardConnection.class,
ServiceType.DASHBOARD);
}
@Override
protected String getServiceType(DashboardService dashboardService) {
return dashboardService.getServiceType().value();
}
}

7
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java
View File

@ -20,7 +20,7 @@ import org.openmetadata.catalog.entity.services.ServiceType;
import org.openmetadata.catalog.resources.services.database.DatabaseServiceResource;
import org.openmetadata.catalog.secrets.SecretsManager;
public class DatabaseServiceRepository extends ServiceRepository<DatabaseService, DatabaseConnection> {
public class DatabaseServiceRepository extends ServiceEntityRepository<DatabaseService, DatabaseConnection> {
public DatabaseServiceRepository(CollectionDAO dao, SecretsManager secretsManager) {
super(
DatabaseServiceResource.COLLECTION_PATH,
@ -31,9 +31,4 @@ public class DatabaseServiceRepository extends ServiceRepository<DatabaseService
DatabaseConnection.class,
ServiceType.DATABASE);
}
@Override
protected String getServiceType(DatabaseService databaseService) {
return databaseService.getServiceType().value();
}
}

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/EntityDAO.java
View File

@ -115,6 +115,10 @@ public interface EntityDAO<T extends EntityInterface> {
update(getTableName(), id.toString(), json);
}
default void update(EntityInterface entity) throws JsonProcessingException {
update(getTableName(), entity.getId().toString(), JsonUtils.pojoToJson(entity));
}
default String getCondition(Include include) {
if (!supportsSoftDelete()) {
return "";

7
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java
View File

@ -20,7 +20,7 @@ import org.openmetadata.catalog.resources.services.messaging.MessagingServiceRes
import org.openmetadata.catalog.secrets.SecretsManager;
import org.openmetadata.catalog.type.MessagingConnection;
public class MessagingServiceRepository extends ServiceRepository<MessagingService, MessagingConnection> {
public class MessagingServiceRepository extends ServiceEntityRepository<MessagingService, MessagingConnection> {
private static final String UPDATE_FIELDS = "owner, connection";
public MessagingServiceRepository(CollectionDAO dao, SecretsManager secretsManager) {
@ -34,9 +34,4 @@ public class MessagingServiceRepository extends ServiceRepository<MessagingServi
UPDATE_FIELDS,
ServiceType.MESSAGING);
}
@Override
protected String getServiceType(MessagingService messagingService) {
return messagingService.getServiceType().value();
}
}

7
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelServiceRepository.java
View File

@ -20,7 +20,7 @@ import org.openmetadata.catalog.resources.services.mlmodel.MlModelServiceResourc
import org.openmetadata.catalog.secrets.SecretsManager;
import org.openmetadata.catalog.type.MlModelConnection;
public class MlModelServiceRepository extends ServiceRepository<MlModelService, MlModelConnection> {
public class MlModelServiceRepository extends ServiceEntityRepository<MlModelService, MlModelConnection> {
private static final String UPDATE_FIELDS = "owner,connection";
public MlModelServiceRepository(CollectionDAO dao, SecretsManager secretsManager) {
@ -34,9 +34,4 @@ public class MlModelServiceRepository extends ServiceRepository<MlModelService,
UPDATE_FIELDS,
ServiceType.ML_MODEL);
}
@Override
protected String getServiceType(MlModelService mlModelService) {
return mlModelService.getServiceType().value();
}
}

7
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java
View File

@ -20,7 +20,7 @@ import org.openmetadata.catalog.resources.services.pipeline.PipelineServiceResou
import org.openmetadata.catalog.secrets.SecretsManager;
import org.openmetadata.catalog.type.PipelineConnection;
public class PipelineServiceRepository extends ServiceRepository<PipelineService, PipelineConnection> {
public class PipelineServiceRepository extends ServiceEntityRepository<PipelineService, PipelineConnection> {
public PipelineServiceRepository(CollectionDAO dao, SecretsManager secretsManager) {
super(
@ -32,9 +32,4 @@ public class PipelineServiceRepository extends ServiceRepository<PipelineService
PipelineConnection.class,
ServiceType.PIPELINE);
}
@Override
protected String getServiceType(PipelineService pipelineService) {
return pipelineService.getServiceType().value();
}
}

48
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ServiceRepository.java → catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ServiceEntityRepository.java
View File

@ -1,9 +1,22 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import static org.openmetadata.catalog.Entity.FIELD_OWNER;
import static org.openmetadata.catalog.util.EntityUtil.objectMatch;
import java.io.IOException;
import lombok.Getter;
import org.openmetadata.catalog.ServiceConnectionEntityInterface;
import org.openmetadata.catalog.ServiceEntityInterface;
import org.openmetadata.catalog.entity.services.ServiceType;
@ -12,17 +25,18 @@ import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.JsonUtils;
public abstract class ServiceRepository<T extends ServiceEntityInterface, S extends ServiceConnectionEntityInterface>
public abstract class ServiceEntityRepository<
T extends ServiceEntityInterface, S extends ServiceConnectionEntityInterface>
extends EntityRepository<T> {
private static final String UPDATE_FIELDS = "owner";
private final Class<S> serviceConnectionClass;
@Getter private final Class<S> serviceConnectionClass;
protected final SecretsManager secretsManager;
private final ServiceType serviceType;
@Getter private final ServiceType serviceType;
protected ServiceRepository(
protected ServiceEntityRepository(
String collectionPath,
String service,
CollectionDAO dao,
@ -37,7 +51,7 @@ public abstract class ServiceRepository<T extends ServiceEntityInterface, S exte
this.serviceType = serviceType;
}
protected ServiceRepository(
protected ServiceEntityRepository(
String collectionPath,
String service,
CollectionDAO dao,
@ -65,8 +79,6 @@ public abstract class ServiceRepository<T extends ServiceEntityInterface, S exte
setFullyQualifiedName(service);
}
protected abstract String getServiceType(T service);
@Override
public void storeEntity(T service, boolean update) throws IOException {
// Relationships and fields such as href are derived and not stored as part of json
@ -81,7 +93,11 @@ public abstract class ServiceRepository<T extends ServiceEntityInterface, S exte
.getConnection()
.setConfig(
secretsManager.encryptOrDecryptServiceConnectionConfig(
service.getConnection().getConfig(), getServiceType(service), service.getName(), serviceType, true));
service.getConnection().getConfig(),
service.getServiceType().value(),
service.getName(),
serviceType,
true));
store(service.getId(), service, update);
} else {
// otherwise, nullify the config since it will be kept outside OM
@ -93,7 +109,7 @@ public abstract class ServiceRepository<T extends ServiceEntityInterface, S exte
.getConnection()
.setConfig(
secretsManager.encryptOrDecryptServiceConnectionConfig(
connectionConfig, getServiceType(service), service.getName(), serviceType, true));
connectionConfig, service.getServiceType().value(), service.getName(), serviceType, true));
}
// Restore the relationships
@ -132,10 +148,18 @@ public abstract class ServiceRepository<T extends ServiceEntityInterface, S exte
S decryptedUpdatedConn = JsonUtils.readValue(updatedJson, serviceConnectionClass);
decryptedOrigConn.setConfig(
secretsManager.encryptOrDecryptServiceConnectionConfig(
decryptedOrigConn.getConfig(), getServiceType(original), original.getName(), serviceType, false));
decryptedOrigConn.getConfig(),
original.getServiceType().value(),
original.getName(),
serviceType,
false));
decryptedUpdatedConn.setConfig(
secretsManager.encryptOrDecryptServiceConnectionConfig(
decryptedUpdatedConn.getConfig(), getServiceType(updated), updated.getName(), serviceType, false));
decryptedUpdatedConn.getConfig(),
updated.getServiceType().value(),
updated.getName(),
serviceType,
false));
if (!objectMatch.test(decryptedOrigConn, decryptedUpdatedConn)) {
recordChange("connection", origConn, updatedConn, true);
}
@ -145,7 +169,7 @@ public abstract class ServiceRepository<T extends ServiceEntityInterface, S exte
.setConfig(
secretsManager.encryptOrDecryptServiceConnectionConfig(
original.getConnection().getConfig(),
getServiceType(original),
original.getServiceType().value(),
original.getName(),
serviceType,
false));

10
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/CollectionRegistry.java
View File

@ -29,6 +29,8 @@ import java.util.Objects;
import java.util.Set;
import javax.ws.rs.Path;
import javax.ws.rs.core.UriInfo;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Jdbi;
import org.openmetadata.catalog.CatalogApplicationConfig;
@ -80,7 +82,7 @@ public final class CollectionRegistry {
return children;
}
Map<String, CollectionDetails> getCollectionMap() {
public Map<String, CollectionDetails> getCollectionMap() {
return Collections.unmodifiableMap(collectionMap);
}
@ -135,6 +137,7 @@ public final class CollectionRegistry {
CollectionDAO daoObject = jdbi.onDemand(CollectionDAO.class);
Objects.requireNonNull(daoObject, "CollectionDAO must not be null");
Object resource = createResource(daoObject, resourceClass, config, authorizer, secretsManager);
details.setResource(resource);
environment.jersey().register(resource);
LOG.info("Registering {} with order {}", resourceClass, details.order);
} catch (Exception ex) {
@ -232,10 +235,13 @@ public final class CollectionRegistry {
public static class CollectionDetails {
private final int order;
private final String resourceClass;
@Getter private final String resourceClass;
private final CollectionDescriptor cd;
private final List<CollectionDescriptor> childCollections = new ArrayList<>();
@Getter @Setter private Object resource;
CollectionDetails(CollectionDescriptor cd, String resourceClass, int order) {
this.cd = cd;
this.resourceClass = resourceClass;

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/events/EventResource.java
View File

@ -31,6 +31,7 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.UriInfo;
import lombok.Getter;
import org.openmetadata.catalog.Entity.EntityList;
import org.openmetadata.catalog.jdbi3.ChangeEventRepository;
import org.openmetadata.catalog.jdbi3.CollectionDAO;
@ -46,7 +47,8 @@ import org.openmetadata.catalog.util.ResultList;
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "events")
public class EventResource {
private final ChangeEventRepository dao;
@Getter private final ChangeEventRepository dao;
private final Authorizer authorizer;
public static class ChangeEventList extends ResultList<ChangeEvent> {

23
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ServiceEntityResource.java
View File

@ -1,12 +1,26 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.resources.services;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import javax.ws.rs.core.SecurityContext;
import lombok.Getter;
import org.openmetadata.catalog.ServiceConnectionEntityInterface;
import org.openmetadata.catalog.ServiceEntityInterface;
import org.openmetadata.catalog.entity.services.ServiceType;
import org.openmetadata.catalog.jdbi3.ServiceRepository;
import org.openmetadata.catalog.jdbi3.ServiceEntityRepository;
import org.openmetadata.catalog.resources.EntityResource;
import org.openmetadata.catalog.secrets.SecretsManager;
import org.openmetadata.catalog.security.AuthorizationException;
@ -14,11 +28,15 @@ import org.openmetadata.catalog.security.Authorizer;
import org.openmetadata.catalog.util.ResultList;
public abstract class ServiceEntityResource<
T extends ServiceEntityInterface, R extends ServiceRepository<T, S>, S extends ServiceConnectionEntityInterface>
T extends ServiceEntityInterface,
R extends ServiceEntityRepository<T, S>,
S extends ServiceConnectionEntityInterface>
extends EntityResource<T, R> {
private final SecretsManager secretsManager;
@Getter private final ServiceEntityRepository<T, S> serviceEntityRepository;
private final ServiceType serviceType;
protected ServiceEntityResource(
@ -28,6 +46,7 @@ public abstract class ServiceEntityResource<
SecretsManager secretsManager,
ServiceType serviceType) {
super(entityClass, serviceRepository, authorizer);
this.serviceEntityRepository = serviceRepository;
this.secretsManager = secretsManager;
this.serviceType = serviceType;
}

4
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java
View File

@ -51,6 +51,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.catalog.CatalogApplicationConfig;
import org.openmetadata.catalog.Entity;
@ -84,6 +85,8 @@ public class IngestionPipelineResource extends EntityResource<IngestionPipeline,
private CatalogApplicationConfig catalogApplicationConfig;
private final SecretsManager secretsManager;
@Getter private final IngestionPipelineRepository ingestionPipelineRepository;
@Override
public IngestionPipeline addHref(UriInfo uriInfo, IngestionPipeline ingestionPipeline) {
Entity.withHref(uriInfo, ingestionPipeline.getOwner());
@ -94,6 +97,7 @@ public class IngestionPipelineResource extends EntityResource<IngestionPipeline,
public IngestionPipelineResource(CollectionDAO dao, Authorizer authorizer, SecretsManager secretsManager) {
super(IngestionPipeline.class, new IngestionPipelineRepository(dao), authorizer);
this.secretsManager = secretsManager;
this.ingestionPipelineRepository = new IngestionPipelineRepository(dao);
}
public void initialize(CatalogApplicationConfig config) {

57
catalog-rest-service/src/main/java/org/openmetadata/catalog/secrets/AWSSecretsManager.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.secrets;
import static org.openmetadata.catalog.services.connections.metadata.OpenMetadataServerConnection.SecretsManagerProvider.AWS;
@ -23,32 +36,32 @@ import software.amazon.awssdk.services.secretsmanager.model.UpdateSecretRequest;
public class AWSSecretsManager extends SecretsManager {
public static final String AUTH_PROVIDER_SECRET_ID_SUFFIX = "auth-provider";
public static final String ACCESS_KEY_ID = "accessKeyId";
public static final String SECRET_ACCESS_KEY = "secretAccessKey";
public static final String REGION = "region";
private static AWSSecretsManager INSTANCE = null;
private SecretsManagerClient secretsClient;
private AWSSecretsManager(
OpenMetadataServerConnection.SecretsManagerProvider secretsManagerProvider,
SecretsManagerConfiguration config,
String clusterPrefix) {
super(secretsManagerProvider, clusterPrefix);
if (config == null) {
throw new SecretsManagerException("Secrets manager configuration is empty.");
private AWSSecretsManager(SecretsManagerConfiguration config, String clusterPrefix) {
super(AWS, clusterPrefix);
// initialize the secret client depending on the SecretsManagerConfiguration passed
if (config != null && config.getParameters() != null) {
String accessKeyId = config.getParameters().getOrDefault(ACCESS_KEY_ID, "");
String secretAccessKey = config.getParameters().getOrDefault(SECRET_ACCESS_KEY, "");
String region = config.getParameters().getOrDefault(REGION, "");
this.secretsClient =
SecretsManagerClient.builder()
.region(Region.of(region))
.credentialsProvider(
StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey)))
.build();
} else {
// initialized with the region loaded from the DefaultAwsRegionProviderChain and credentials loaded from the
// DefaultCredentialsProvider
this.secretsClient = SecretsManagerClient.create();
}
String region = config.getParameters().getOrDefault("region", "");
String accessKeyId = config.getParameters().getOrDefault("accessKeyId", "");
String secretAccessKey = config.getParameters().getOrDefault("secretAccessKey", "");
this.secretsClient =
SecretsManagerClient.builder()
.region(Region.of(region))
.credentialsProvider(
StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey)))
.build();
}
@Override
public boolean isLocal() {
return false;
}
@Override
@ -164,7 +177,7 @@ public class AWSSecretsManager extends SecretsManager {
}
public static AWSSecretsManager getInstance(SecretsManagerConfiguration config, String clusterPrefix) {
if (INSTANCE == null) INSTANCE = new AWSSecretsManager(AWS, config, clusterPrefix);
if (INSTANCE == null) INSTANCE = new AWSSecretsManager(config, clusterPrefix);
return INSTANCE;
}

20
catalog-rest-service/src/main/java/org/openmetadata/catalog/secrets/LocalSecretsManager.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.secrets;
import static org.openmetadata.catalog.services.connections.metadata.OpenMetadataServerConnection.SecretsManagerProvider.LOCAL;
@ -19,9 +32,8 @@ public class LocalSecretsManager extends SecretsManager {
private Fernet fernet;
private LocalSecretsManager(
OpenMetadataServerConnection.SecretsManagerProvider secretsManagerProvider, String clusterPrefix) {
super(secretsManagerProvider, clusterPrefix);
private LocalSecretsManager(String clusterPrefix) {
super(LOCAL, clusterPrefix);
this.fernet = Fernet.getInstance();
}
@ -91,7 +103,7 @@ public class LocalSecretsManager extends SecretsManager {
}
public static LocalSecretsManager getInstance(String clusterPrefix) {
if (INSTANCE == null) INSTANCE = new LocalSecretsManager(LOCAL, clusterPrefix);
if (INSTANCE == null) INSTANCE = new LocalSecretsManager(clusterPrefix);
return INSTANCE;
}

17
catalog-rest-service/src/main/java/org/openmetadata/catalog/secrets/SecretsManager.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.secrets;
import static java.util.Objects.isNull;
@ -23,7 +36,9 @@ public abstract class SecretsManager {
this.clusterPrefix = clusterPrefix;
}
public abstract boolean isLocal();
public boolean isLocal() {
return false;
}
public abstract Object encryptOrDecryptServiceConnectionConfig(
Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType, boolean encrypt);

13
catalog-rest-service/src/main/java/org/openmetadata/catalog/secrets/SecretsManagerConfiguration.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.secrets;
import java.util.Map;

13
catalog-rest-service/src/main/java/org/openmetadata/catalog/secrets/SecretsManagerFactory.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.secrets;
import org.openmetadata.catalog.services.connections.metadata.OpenMetadataServerConnection.SecretsManagerProvider;

307
catalog-rest-service/src/main/java/org/openmetadata/catalog/secrets/SecretsManagerMigrationService.java Normal file
View File

@ -0,0 +1,307 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.secrets;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.ServiceConnectionEntityInterface;
import org.openmetadata.catalog.ServiceEntityInterface;
import org.openmetadata.catalog.entity.services.ingestionPipelines.IngestionPipeline;
import org.openmetadata.catalog.exception.SecretsManagerMigrationException;
import org.openmetadata.catalog.jdbi3.ChangeEventRepository;
import org.openmetadata.catalog.jdbi3.IngestionPipelineRepository;
import org.openmetadata.catalog.jdbi3.ListFilter;
import org.openmetadata.catalog.jdbi3.ServiceEntityRepository;
import org.openmetadata.catalog.resources.CollectionRegistry;
import org.openmetadata.catalog.resources.events.EventResource;
import org.openmetadata.catalog.resources.services.ServiceEntityResource;
import org.openmetadata.catalog.resources.services.ingestionpipelines.IngestionPipelineResource;
import org.openmetadata.catalog.util.EntityUtil;
/**
* Migration service from LocalSecretManager to configured one.
*
* <p>It will migrate all the entities with connection parameters:
*
* <p>- the connection config objects of services entities which implement the ServiceEntityResource and
* ServiceEntityRepository (services using the secrets' manager)
*
* <p>- remove the auth security config in the IngestionPipeline entities
*
* <p>- remove all the ChangeEvent entities related to ingestion pipelines and services
*/
@Slf4j
public class SecretsManagerMigrationService {
private final SecretsManager newSecretManager;
private final SecretsManager oldSecretManager;
private final Map<Class<? extends ServiceConnectionEntityInterface>, ServiceEntityRepository<?, ?>>
connectionTypeRepositoriesMap;
private final ChangeEventRepository changeEventRepository;
private final IngestionPipelineRepository ingestionPipelineRepository;
public SecretsManagerMigrationService(SecretsManager secretsManager, String clusterName) {
this.newSecretManager = secretsManager;
this.connectionTypeRepositoriesMap = retrieveConnectionTypeRepositoriesMap();
this.changeEventRepository = retrieveChangeEventRepository();
this.ingestionPipelineRepository = retrieveIngestionPipelineRepository();
// by default, it is going to be LOCAL
this.oldSecretManager = SecretsManagerFactory.createSecretsManager(null, clusterName);
}
public void migrateServicesToSecretManagerIfNeeded() {
if (!newSecretManager.isLocal()) {
migrateServices();
migrateIngestionPipelines();
} else {
LOG.info("Local secrets manager does not need to check if migration is needed.");
}
}
private void migrateServices() {
LOG.info(
String.format(
"Checking if services migration is needed for secrets manager: [{}]",
newSecretManager.getSecretsManagerProvider()));
List<ServiceEntityInterface> notStoredServices = retrieveNotStoredServices();
if (!notStoredServices.isEmpty()) {
notStoredServices.forEach(this::migrateService);
deleteChangeEventsForServices();
} else {
LOG.info(
String.format(
"All services are already safely stored in [{}] secrets manager",
newSecretManager.getSecretsManagerProvider()));
}
}
private void migrateIngestionPipelines() {
LOG.info(
String.format(
"Checking if ingestion pipelines migration is needed for secrets manager: [{}]",
newSecretManager.getSecretsManagerProvider()));
List<IngestionPipeline> notStoredIngestionPipelines = retrieveNotStoredIngestionPipelines();
if (!notStoredIngestionPipelines.isEmpty()) {
notStoredIngestionPipelines.forEach(this::migrateIngestionPipelines);
deleteChangeEventsForIngestionPipelines();
} else {
LOG.info(
String.format(
"All ingestion pipelines are already safely stored in [{}] secrets manager",
newSecretManager.getSecretsManagerProvider()));
}
}
private void migrateService(ServiceEntityInterface serviceEntityInterface) {
ServiceEntityRepository<?, ?> repository =
connectionTypeRepositoriesMap.get(serviceEntityInterface.getConnection().getClass());
try {
ServiceEntityInterface service = repository.dao.findEntityById(serviceEntityInterface.getId());
// we have to decrypt using the old secrets manager and encrypt again with the new one
service
.getConnection()
.setConfig(
oldSecretManager.encryptOrDecryptServiceConnectionConfig(
service.getConnection().getConfig(),
service.getServiceType().value(),
service.getName(),
repository.getServiceType(),
false));
newSecretManager.encryptOrDecryptServiceConnectionConfig(
service.getConnection().getConfig(),
service.getServiceType().value(),
service.getName(),
repository.getServiceType(),
true);
// avoid reaching secrets manager quotas
Thread.sleep(100);
service.getConnection().setConfig(null);
repository.dao.update(service);
} catch (IOException | InterruptedException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
}
}
private List<ServiceEntityInterface> retrieveNotStoredServices() {
return connectionTypeRepositoriesMap.values().stream()
.map(this::retrieveServices)
.flatMap(List<ServiceEntityInterface>::stream)
.collect(Collectors.toList());
}
private List<ServiceEntityInterface> retrieveServices(ServiceEntityRepository<?, ?> serviceEntityRepository) {
try {
return serviceEntityRepository
.listAfter(
null,
EntityUtil.Fields.EMPTY_FIELDS,
new ListFilter(),
serviceEntityRepository.dao.listCount(new ListFilter()),
null)
.getData().stream()
.map(ServiceEntityInterface.class::cast)
.filter(
service ->
!Objects.isNull(service.getConnection()) && !Objects.isNull(service.getConnection().getConfig()))
.collect(Collectors.toList());
} catch (IOException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
}
}
private void migrateIngestionPipelines(IngestionPipeline ingestionPipeline) {
try {
IngestionPipeline ingestion = ingestionPipelineRepository.dao.findEntityById(ingestionPipeline.getId());
ingestion.getOpenMetadataServerConnection().setSecurityConfig(null);
ingestionPipelineRepository.dao.update(ingestion);
} catch (IOException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
}
}
private List<IngestionPipeline> retrieveNotStoredIngestionPipelines() {
try {
return ingestionPipelineRepository
.listAfter(
null,
EntityUtil.Fields.EMPTY_FIELDS,
new ListFilter(),
ingestionPipelineRepository.dao.listCount(new ListFilter()),
null)
.getData().stream()
.filter(
ingestionPipeline ->
!Objects.isNull(ingestionPipeline.getOpenMetadataServerConnection())
&& !Objects.isNull(ingestionPipeline.getOpenMetadataServerConnection().getSecurityConfig()))
.collect(Collectors.toList());
} catch (IOException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
}
}
/** This method delete all the change events which could contain connection config parameters for services */
private void deleteChangeEventsForServices() {
connectionTypeRepositoriesMap.values().stream()
.map(ServiceEntityRepository::getServiceType)
.forEach(
serviceType -> {
try {
changeEventRepository.deleteAll(
Entity.class
.getField(serviceType.value().toUpperCase(Locale.ROOT) + "_SERVICE")
.get(Entity.class)
.toString());
} catch (NoSuchFieldException | IOException | IllegalAccessException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
}
});
}
/**
* This method delete all the change events which could contain auth provider config parameters for ingestion
* pipelines
*/
private void deleteChangeEventsForIngestionPipelines() {
ChangeEventRepository changeEventRepository = retrieveChangeEventRepository();
try {
changeEventRepository.deleteAll(Entity.INGESTION_PIPELINE);
} catch (IOException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
}
}
private Map<Class<? extends ServiceConnectionEntityInterface>, ServiceEntityRepository<?, ?>>
retrieveConnectionTypeRepositoriesMap() {
Map<Class<? extends ServiceConnectionEntityInterface>, ServiceEntityRepository<?, ?>>
connectionTypeRepositoriesMap =
CollectionRegistry.getInstance().getCollectionMap().values().stream()
.map(this::retrieveServiceRepository)
.filter(Optional::isPresent)
.map(Optional::get)
.collect(Collectors.toMap(ServiceEntityRepository::getServiceConnectionClass, Function.identity()));
if (connectionTypeRepositoriesMap.isEmpty()) {
throw new SecretsManagerMigrationException("Unexpected error: ServiceRepository not found.");
}
return connectionTypeRepositoriesMap;
}
private ChangeEventRepository retrieveChangeEventRepository() {
return CollectionRegistry.getInstance().getCollectionMap().values().stream()
.map(this::retrieveChangeEventRepository)
.filter(Optional::isPresent)
.map(Optional::get)
.findFirst()
.orElseThrow(() -> new SecretsManagerMigrationException("Unexpected error: ChangeEventRepository not found."));
}
private IngestionPipelineRepository retrieveIngestionPipelineRepository() {
return CollectionRegistry.getInstance().getCollectionMap().values().stream()
.map(this::retrieveIngestionPipelineRepository)
.filter(Optional::isPresent)
.map(Optional::get)
.findFirst()
.orElseThrow(
() -> new SecretsManagerMigrationException("Unexpected error: IngestionPipelineRepository not found."));
}
private Optional<IngestionPipelineRepository> retrieveIngestionPipelineRepository(
CollectionRegistry.CollectionDetails collectionDetails) {
Class<?> collectionDetailsClass = extractCollectionDetailsClass(collectionDetails);
if (IngestionPipelineResource.class.equals(collectionDetailsClass)) {
return Optional.of(
((IngestionPipelineResource) collectionDetails.getResource()).getIngestionPipelineRepository());
}
return Optional.empty();
}
private Optional<ChangeEventRepository> retrieveChangeEventRepository(
CollectionRegistry.CollectionDetails collectionDetails) {
Class<?> collectionDetailsClass = extractCollectionDetailsClass(collectionDetails);
if (EventResource.class.equals(collectionDetailsClass)) {
return Optional.of(((EventResource) collectionDetails.getResource()).getDao());
}
return Optional.empty();
}
private Optional<ServiceEntityRepository<?, ?>> retrieveServiceRepository(
CollectionRegistry.CollectionDetails collectionDetails) {
Class<?> collectionDetailsClass = extractCollectionDetailsClass(collectionDetails);
if (ServiceEntityResource.class.isAssignableFrom(collectionDetailsClass)) {
return Optional.of(
((ServiceEntityResource<?, ?, ?>) collectionDetails.getResource()).getServiceEntityRepository());
}
return Optional.empty();
}
private Class<?> extractCollectionDetailsClass(CollectionRegistry.CollectionDetails collectionDetails) {
Class<?> collectionDetailsClass;
try {
collectionDetailsClass = Class.forName(collectionDetails.getResourceClass());
} catch (ClassNotFoundException e) {
throw new SecretsManagerMigrationException(e.getMessage(), e.getCause());
}
return collectionDetailsClass;
}
}

1
catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json
View File

@ -13,6 +13,7 @@
"dashboardServiceType": {
"description": "Type of Dashboard service - Superset, Looker, Redash, Tableau, Metabase, PowerBi or Mode",
"type": "string",
"javaInterfaces": ["org.openmetadata.catalog.EnumInterface"],
"enum": [
"Superset",
"Looker",

1
catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json
View File

@ -12,6 +12,7 @@
"definitions": {
"databaseServiceType": {
"description": "Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres...",
"javaInterfaces": ["org.openmetadata.catalog.EnumInterface"],
"type": "string",
"enum": [
"BigQuery",

1
catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json
View File

@ -13,6 +13,7 @@
"messagingServiceType": {
"description": "Type of messaging service - Kafka or Pulsar.",
"type": "string",
"javaInterfaces": ["org.openmetadata.catalog.EnumInterface"],
"enum": ["Kafka", "Pulsar"],
"javaEnums": [
{

1
catalog-rest-service/src/main/resources/json/schema/entity/services/mlmodelService.json
View File

@ -13,6 +13,7 @@
"mlModelServiceType": {
"description": "Type of MlModel service",
"type": "string",
"javaInterfaces": ["org.openmetadata.catalog.EnumInterface"],
"enum": ["Mlflow", "Sklearn"],
"javaEnums": [
{

1
catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json
View File

@ -13,6 +13,7 @@
"pipelineServiceType": {
"description": "Type of pipeline service - Airflow or Prefect.",
"type": "string",
"javaInterfaces": ["org.openmetadata.catalog.EnumInterface"],
"enum": ["Airflow", "Glue", "Airbyte", "Fivetran"],
"javaEnums": [
{

15
catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepositoryUnitTest.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import static org.mockito.ArgumentMatchers.isNull;
@ -12,7 +25,7 @@ import org.openmetadata.catalog.services.connections.database.MysqlConnection;
import org.openmetadata.catalog.type.DashboardConnection;
public class DashboardServiceRepositoryUnitTest
extends ServiceRepositoryTest<DashboardServiceRepository, DashboardService, DashboardConnection> {
extends ServiceEntityRepositoryTest<DashboardServiceRepository, DashboardService, DashboardConnection> {
protected DashboardServiceRepositoryUnitTest() {
super(ServiceType.DASHBOARD);

15
catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepositoryUnitTest.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import static org.mockito.ArgumentMatchers.isNull;
@ -12,7 +25,7 @@ import org.openmetadata.catalog.secrets.SecretsManager;
import org.openmetadata.catalog.services.connections.database.MysqlConnection;
public class DatabaseServiceRepositoryUnitTest
extends ServiceRepositoryTest<DatabaseServiceRepository, DatabaseService, DatabaseConnection> {
extends ServiceEntityRepositoryTest<DatabaseServiceRepository, DatabaseService, DatabaseConnection> {
protected DatabaseServiceRepositoryUnitTest() {
super(ServiceType.DATABASE);

15
catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepositoryUnitTest.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import static org.mockito.ArgumentMatchers.isNull;
@ -12,7 +25,7 @@ import org.openmetadata.catalog.services.connections.database.MysqlConnection;
import org.openmetadata.catalog.type.MessagingConnection;
public class MessagingServiceRepositoryUnitTest
extends ServiceRepositoryTest<MessagingServiceRepository, MessagingService, MessagingConnection> {
extends ServiceEntityRepositoryTest<MessagingServiceRepository, MessagingService, MessagingConnection> {
protected MessagingServiceRepositoryUnitTest() {
super(ServiceType.MESSAGING);

15
catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/MlModelServiceRepositoryUnitTest.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import static org.mockito.ArgumentMatchers.isNull;
@ -12,7 +25,7 @@ import org.openmetadata.catalog.services.connections.database.MysqlConnection;
import org.openmetadata.catalog.type.MlModelConnection;
public class MlModelServiceRepositoryUnitTest
extends ServiceRepositoryTest<MlModelServiceRepository, MlModelService, MlModelConnection> {
extends ServiceEntityRepositoryTest<MlModelServiceRepository, MlModelService, MlModelConnection> {
protected MlModelServiceRepositoryUnitTest() {
super(ServiceType.ML_MODEL);

15
catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepositoryUnitTest.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import static org.mockito.ArgumentMatchers.isNull;
@ -12,7 +25,7 @@ import org.openmetadata.catalog.services.connections.database.MysqlConnection;
import org.openmetadata.catalog.type.PipelineConnection;
public class PipelineServiceRepositoryUnitTest
extends ServiceRepositoryTest<PipelineServiceRepository, PipelineService, PipelineConnection> {
extends ServiceEntityRepositoryTest<PipelineServiceRepository, PipelineService, PipelineConnection> {
protected PipelineServiceRepositoryUnitTest() {
super(ServiceType.PIPELINE);

21
catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/ServiceRepositoryTest.java → catalog-rest-service/src/test/java/org/openmetadata/catalog/jdbi3/ServiceEntityRepositoryTest.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.jdbi3;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -23,8 +36,10 @@ import org.openmetadata.catalog.entity.services.ServiceType;
import org.openmetadata.catalog.secrets.SecretsManager;
@ExtendWith(MockitoExtension.class)
public abstract class ServiceRepositoryTest<
T extends ServiceRepository<R, S>, R extends ServiceEntityInterface, S extends ServiceConnectionEntityInterface> {
public abstract class ServiceEntityRepositoryTest<
T extends ServiceEntityRepository<R, S>,
R extends ServiceEntityInterface,
S extends ServiceConnectionEntityInterface> {
@Mock protected CollectionDAO collectionDAO;
@ -38,7 +53,7 @@ public abstract class ServiceRepositoryTest<
private final ServiceType expectedServiceType;
protected ServiceRepositoryTest(ServiceType serviceType) {
protected ServiceEntityRepositoryTest(ServiceType serviceType) {
this.expectedServiceType = serviceType;
}

17
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ServiceResourceTest.java
View File

@ -1,3 +1,16 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.resources.services;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -25,7 +38,7 @@ import org.openmetadata.catalog.ServiceConnectionEntityInterface;
import org.openmetadata.catalog.ServiceEntityInterface;
import org.openmetadata.catalog.entity.services.ServiceType;
import org.openmetadata.catalog.jdbi3.CollectionDAO;
import org.openmetadata.catalog.jdbi3.ServiceRepository;
import org.openmetadata.catalog.jdbi3.ServiceEntityRepository;
import org.openmetadata.catalog.secrets.SecretsManager;
import org.openmetadata.catalog.security.AuthorizationException;
import org.openmetadata.catalog.security.Authorizer;
@ -34,7 +47,7 @@ import org.openmetadata.catalog.security.Authorizer;
public abstract class ServiceResourceTest<
T extends ServiceEntityResource<R, S, U>,
R extends ServiceEntityInterface,
S extends ServiceRepository<R, U>,
S extends ServiceEntityRepository<R, U>,
U extends ServiceConnectionEntityInterface> {
T serviceResource;

12
catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/AWSSecretsManagerTest.java
View File

@ -1,3 +1,15 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.secrets;
import static org.junit.jupiter.api.Assertions.assertEquals;

12
catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/LocalSecretsManagerTest.java
View File

@ -1,3 +1,15 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.secrets;
import static org.junit.jupiter.api.Assertions.assertEquals;

12
catalog-rest-service/src/test/java/org/openmetadata/catalog/secrets/SecretsManagerFactoryTest.java
View File

@ -1,3 +1,15 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.catalog.secrets;
import static org.junit.jupiter.api.Assertions.assertTrue;

1
conf/openmetadata.yaml
View File

@ -228,6 +228,7 @@ fernetConfiguration:
secretsManagerConfiguration:
secretsManager: ${SECRET_MANAGER:-local} # Possible values are "local", "aws"
# secretsManager: aws
# it will use the default auth provider for the secrets' manager service if parameters are not set
# parameters:
# region:
# accessKeyId:

19
openmetadata-core/src/main/java/org/openmetadata/core/entity/interfaces/EnumInterface.java Normal file
View File

@ -0,0 +1,19 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openmetadata.core.entity.interfaces;
/** Interface which could be implemented by Enums classes */
public interface EnumInterface {
String value();
}

3
openmetadata-core/src/main/resources/json/schema/entity/services/dashboardService.json
View File

@ -10,6 +10,9 @@
"dashboardServiceType": {
"description": "Type of Dashboard service - Superset, Looker, Redash, Tableau, Metabase, PowerBi or Mode",
"type": "string",
"javaInterfaces": [
"org.openmetadata.core.entity.interfaces.EnumInterface"
],
"enum": [
"Superset",
"Looker",

3
openmetadata-core/src/main/resources/json/schema/entity/services/databaseService.json
View File

@ -10,6 +10,9 @@
"databaseServiceType": {
"description": "Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres...",
"type": "string",
"javaInterfaces": [
"org.openmetadata.core.entity.interfaces.EnumInterface"
],
"enum": [
"BigQuery",
"Mysql",

3
openmetadata-core/src/main/resources/json/schema/entity/services/messagingService.json
View File

@ -10,6 +10,9 @@
"messagingServiceType": {
"description": "Type of messaging service - Kafka or Pulsar.",
"type": "string",
"javaInterfaces": [
"org.openmetadata.core.entity.interfaces.EnumInterface"
],
"enum": ["Kafka", "Pulsar"],
"javaEnums": [
{

3
openmetadata-core/src/main/resources/json/schema/entity/services/mlmodelService.json
View File

@ -10,6 +10,9 @@
"mlModelServiceType": {
"description": "Type of MlModel service",
"type": "string",
"javaInterfaces": [
"org.openmetadata.core.entity.interfaces.EnumInterface"
],
"enum": ["Mlflow", "Sklearn"],
"javaEnums": [
{

3
openmetadata-core/src/main/resources/json/schema/entity/services/pipelineService.json
View File

@ -10,6 +10,9 @@
"pipelineServiceType": {
"description": "Type of pipeline service - Airflow or Prefect.",
"type": "string",
"javaInterfaces": [
"org.openmetadata.core.entity.interfaces.EnumInterface"
],
"enum": ["Airflow", "Prefect", "Glue", "Generic", "Airbyte"],
"javaEnums": [
{