From 3ae2ff1c1c31d5eb0da05fa1664fc2605fa4762f Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 17 Feb 2025 00:30:26 -0800 Subject: [PATCH] Fix install appplication (#19819) --- .../apps/AppMarketPlaceResource.java | 22 +-- .../service/util/AppMarketPlaceUtil.java | 59 ++++++++ .../service/util/OpenMetadataOperations.java | 130 +++++++++++++----- 3 files changed, 157 insertions(+), 54 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java index 374ab5ba719..0400e2dc9f2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java @@ -1,8 +1,5 @@ package org.openmetadata.service.resources.apps; -import static org.openmetadata.service.Entity.APPLICATION; -import static org.openmetadata.service.jdbi3.EntityRepository.getEntitiesFromSeedData; - import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -12,7 +9,6 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; -import java.util.List; import java.util.UUID; import javax.json.JsonPatch; import javax.validation.Valid; @@ -52,6 +48,7 @@ import org.openmetadata.service.limits.Limits; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.util.AppMarketPlaceUtil; import org.openmetadata.service.util.ResultList; @Path("/v1/apps/marketplace") @@ -65,30 +62,17 @@ import org.openmetadata.service.util.ResultList; public class AppMarketPlaceResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/apps/marketplace/"; - private PipelineServiceClientInterface pipelineServiceClient; private AppMarketPlaceMapper mapper; static final String FIELDS = "owners,tags"; @Override public void initialize(OpenMetadataApplicationConfig config) { try { - this.pipelineServiceClient = + PipelineServiceClientInterface pipelineServiceClient = PipelineServiceClientFactory.createPipelineServiceClient( config.getPipelineServiceClientConfiguration()); - mapper = new AppMarketPlaceMapper(pipelineServiceClient); - // Initialize Default Installed Applications - List createAppMarketPlaceDefinitionReqs = - getEntitiesFromSeedData( - APPLICATION, - String.format(".*json/data/%s/.*\\.json$", entityType), - CreateAppMarketPlaceDefinitionReq.class); - for (CreateAppMarketPlaceDefinitionReq definitionReq : createAppMarketPlaceDefinitionReqs) { - AppMarketPlaceDefinition definition = mapper.createToEntity(definitionReq, "admin"); - // Update Fully Qualified Name - repository.setFullyQualifiedName(definition); - this.repository.createOrUpdate(null, definition); - } + AppMarketPlaceUtil.createAppMarketPlaceDefinitions(repository, mapper); } catch (Exception ex) { LOG.error("Failed in initializing App MarketPlace Resource", ex); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java new file mode 100644 index 00000000000..28a5c565764 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java @@ -0,0 +1,59 @@ +package org.openmetadata.service.util; + +import static org.openmetadata.service.Entity.APPLICATION; +import static org.openmetadata.service.jdbi3.AppRepository.APP_BOT_ROLE; +import static org.openmetadata.service.jdbi3.EntityRepository.getEntitiesFromSeedData; + +import java.io.IOException; +import java.util.List; +import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; +import org.openmetadata.schema.entity.app.CreateAppMarketPlaceDefinitionReq; +import org.openmetadata.schema.entity.teams.Role; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.AppMarketPlaceRepository; +import org.openmetadata.service.jdbi3.PolicyRepository; +import org.openmetadata.service.jdbi3.RoleRepository; +import org.openmetadata.service.jdbi3.TeamRepository; +import org.openmetadata.service.resources.apps.AppMarketPlaceMapper; + +public class AppMarketPlaceUtil { + public static void createAppMarketPlaceDefinitions( + AppMarketPlaceRepository appMarketRepository, AppMarketPlaceMapper mapper) + throws IOException { + PolicyRepository policyRepository = Entity.getPolicyRepository(); + RoleRepository roleRepository = Entity.getRoleRepository(); + + try { + roleRepository.findByName(APP_BOT_ROLE, Include.NON_DELETED); + } catch (EntityNotFoundException e) { + policyRepository.initSeedDataFromResources(); + List roles = roleRepository.getEntitiesFromSeedData(); + for (Role role : roles) { + role.setFullyQualifiedName(role.getName()); + List policies = role.getPolicies(); + for (EntityReference policy : policies) { + EntityReference ref = + Entity.getEntityReferenceByName(Entity.POLICY, policy.getName(), Include.NON_DELETED); + policy.setId(ref.getId()); + } + roleRepository.initializeEntity(role); + } + TeamRepository teamRepository = (TeamRepository) Entity.getEntityRepository(Entity.TEAM); + teamRepository.initOrganization(); + } + + List createAppMarketPlaceDefinitionReqs = + getEntitiesFromSeedData( + APPLICATION, + String.format(".*json/data/%s/.*\\.json$", Entity.APP_MARKET_PLACE_DEF), + CreateAppMarketPlaceDefinitionReq.class); + for (CreateAppMarketPlaceDefinitionReq definitionReq : createAppMarketPlaceDefinitionReqs) { + AppMarketPlaceDefinition definition = mapper.createToEntity(definitionReq, "admin"); + appMarketRepository.setFullyQualifiedName(definition); + appMarketRepository.createOrUpdate(null, definition); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index f01feb67997..f98844585fd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -84,6 +84,7 @@ import org.openmetadata.service.jdbi3.locator.ConnectionType; import org.openmetadata.service.migration.api.MigrationWorkflow; import org.openmetadata.service.resources.CollectionRegistry; import org.openmetadata.service.resources.apps.AppMapper; +import org.openmetadata.service.resources.apps.AppMarketPlaceMapper; import org.openmetadata.service.resources.databases.DatasourceConfig; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.secrets.SecretsManager; @@ -329,45 +330,108 @@ public class OpenMetadataOperations implements Callable { public Integer installApp( @Option( names = {"-n", "--name"}, - description = "Number of records to process in each batch.", + description = "The name of the application to install.", + required = true) + String appName, + @Option( + names = {"--force"}, + description = "Forces migrations to be run again, even if they have ran previously", + defaultValue = "false") + boolean force) { + try { + parseConfig(); + AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); + + if (!force && isAppInstalled(appRepository, appName)) { + LOG.info("App already installed."); + return 0; + } + + if (force && deleteApplication(appRepository, appName)) { + LOG.info("App deleted."); + } + + LOG.info("App not installed. Installing..."); + installApplication(appName, appRepository); + LOG.info("App Installed."); + return 0; + } catch (Exception e) { + LOG.error("Install Application Failed", e); + return 1; + } + } + + @Command(name = "delete-app", description = "Delete the installed application.") + public Integer deleteApp( + @Option( + names = {"-n", "--name"}, + description = "The name of the application to install.", required = true) String appName) { try { parseConfig(); - CollectionRegistry.initialize(); - ApplicationHandler.initialize(config); - CollectionRegistry.getInstance().loadSeedData(jdbi, config, null, null, null, true); - ApplicationHandler.initialize(config); - AppScheduler.initialize(config, collectionDAO, searchRepository); - // Instantiate JWT Token Generator - JWTTokenGenerator.getInstance() - .init( - config.getAuthenticationConfiguration().getTokenValidationAlgorithm(), - config.getJwtTokenConfiguration()); - AppMarketPlaceRepository marketPlaceRepository = - (AppMarketPlaceRepository) Entity.getEntityRepository(Entity.APP_MARKET_PLACE_DEF); - AppMarketPlaceDefinition definition = - marketPlaceRepository.getByName(null, appName, marketPlaceRepository.getFields("id")); AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); - CreateApp createApp = - new CreateApp() - .withName(definition.getName()) - .withDescription(definition.getDescription()) - .withDisplayName(definition.getDisplayName()) - .withAppSchedule(new AppSchedule().withScheduleTimeline(ScheduleTimeline.NONE)) - .withAppConfiguration(new AppConfiguration()); - AppMapper appMapper = new AppMapper(); - App entity = appMapper.createToEntity(createApp, ADMIN_USER_NAME); - appRepository.prepareInternal(entity, true); - appRepository.createOrUpdate(null, entity); - LOG.info("App Installed."); + if (deleteApplication(appRepository, appName)) { + LOG.info("App deleted."); + } return 0; } catch (Exception e) { - LOG.error("Install Application Failed ", e); + LOG.error("Delete Application Failed", e); return 1; } } + private boolean isAppInstalled(AppRepository appRepository, String appName) { + try { + appRepository.findByName(appName, Include.NON_DELETED); + return true; + } catch (EntityNotFoundException e) { + return false; + } + } + + private boolean deleteApplication(AppRepository appRepository, String appName) { + try { + appRepository.deleteByName(ADMIN_USER_NAME, appName, true, true); + return true; + } catch (EntityNotFoundException e) { + return false; + } + } + + private void installApplication(String appName, AppRepository appRepository) throws Exception { + PipelineServiceClientInterface pipelineServiceClient = + PipelineServiceClientFactory.createPipelineServiceClient( + config.getPipelineServiceClientConfiguration()); + + JWTTokenGenerator.getInstance() + .init( + config.getAuthenticationConfiguration().getTokenValidationAlgorithm(), + config.getJwtTokenConfiguration()); + + AppMarketPlaceMapper mapper = new AppMarketPlaceMapper(pipelineServiceClient); + AppMarketPlaceRepository appMarketRepository = + (AppMarketPlaceRepository) Entity.getEntityRepository(Entity.APP_MARKET_PLACE_DEF); + + AppMarketPlaceUtil.createAppMarketPlaceDefinitions(appMarketRepository, mapper); + + AppMarketPlaceDefinition definition = + appMarketRepository.getByName(null, appName, appMarketRepository.getFields("id")); + + CreateApp createApp = + new CreateApp() + .withName(definition.getName()) + .withDescription(definition.getDescription()) + .withDisplayName(definition.getDisplayName()) + .withAppSchedule(new AppSchedule().withScheduleTimeline(ScheduleTimeline.NONE)) + .withAppConfiguration(new AppConfiguration()); + + AppMapper appMapper = new AppMapper(); + App entity = appMapper.createToEntity(createApp, ADMIN_USER_NAME); + appRepository.prepareInternal(entity, true); + appRepository.createOrUpdate(null, entity); + } + @Command( name = "check-connection", description = @@ -710,9 +774,8 @@ public class OpenMetadataOperations implements Callable { CollectionRegistry.getInstance().loadSeedData(jdbi, config, null, null, null, true); ApplicationHandler.initialize(config); AppScheduler.initialize(config, collectionDAO, searchRepository); - String appName = "DataInsightsApplication"; return executeDataInsightsReindexApp( - appName, batchSize, recreateIndexes, getBackfillConfiguration(startDate, endDate)); + batchSize, recreateIndexes, getBackfillConfiguration(startDate, endDate)); } catch (Exception e) { LOG.error("Failed to reindex due to ", e); return 1; @@ -734,13 +797,10 @@ public class OpenMetadataOperations implements Callable { } private int executeDataInsightsReindexApp( - String appName, - int batchSize, - boolean recreateIndexes, - BackfillConfiguration backfillConfiguration) { + int batchSize, boolean recreateIndexes, BackfillConfiguration backfillConfiguration) { AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); App originalDataInsightsApp = - appRepository.getByName(null, appName, appRepository.getFields("id")); + appRepository.getByName(null, "DataInsightsApplication", appRepository.getFields("id")); DataInsightsAppConfig storedConfig = JsonUtils.convertValue(