From 4db6dce2f99afda68c81ac98cf22f9d36a6f4a13 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 25 Mar 2024 07:05:13 +0100 Subject: [PATCH] MINOR - Add `/status` endpoint (#15394) * MINOR - Add `/validate` endpoint * rename endpoint * fix test * add new migration * Status * Simplify migration client * Simplify migration client * Simplify migration client --- .../service/OpenMetadataApplication.java | 4 + .../service/jdbi3/CollectionDAO.java | 3 + .../service/jdbi3/MigrationDAO.java | 3 + .../service/jdbi3/SystemRepository.java | 122 ++++++++++++++++++ .../migration/MigrationValidationClient.java | 72 +++++++++++ .../MigrationValidationClientException.java | 44 +++++++ .../resources/system/ConfigResource.java | 6 +- .../resources/system/SystemResource.java | 35 ++++- .../resources/system/SystemResourceTest.java | 14 ++ .../schema/system/validationResponse.json | 53 ++++++++ 10 files changed, 351 insertions(+), 5 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClient.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClientException.java create mode 100644 openmetadata-spec/src/main/resources/json/schema/system/validationResponse.json diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index 83978353c3a..71248a46b10 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -81,9 +81,11 @@ import org.openmetadata.service.exception.OMErrorPageHandler; import org.openmetadata.service.fernet.Fernet; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.MigrationDAO; import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator; import org.openmetadata.service.jdbi3.locator.ConnectionType; import org.openmetadata.service.migration.Migration; +import org.openmetadata.service.migration.MigrationValidationClient; import org.openmetadata.service.migration.api.MigrationWorkflow; import org.openmetadata.service.monitoring.EventMonitor; import org.openmetadata.service.monitoring.EventMonitorConfiguration; @@ -161,6 +163,8 @@ public class OpenMetadataApplication extends Application { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MigrationDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MigrationDAO.java index 3fdf445973b..e47b7417ebb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MigrationDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MigrationDAO.java @@ -129,6 +129,9 @@ public interface MigrationDAO { @RegisterRowMapper(FromServerChangeLogMapper.class) List listMetricsFromDBMigrations(); + @SqlQuery("SELECT version FROM SERVER_CHANGE_LOG") + List getMigrationVersions(); + @Getter @Setter class ServerMigrationSQLTable { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java index 94c92c91908..28f308d8dce 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java @@ -13,16 +13,26 @@ import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.schema.api.configuration.SlackAppConfiguration; import org.openmetadata.schema.email.SmtpSettings; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineServiceClientResponse; +import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; import org.openmetadata.schema.settings.Settings; import org.openmetadata.schema.settings.SettingsType; +import org.openmetadata.schema.system.StepValidation; +import org.openmetadata.schema.system.ValidationResponse; import org.openmetadata.schema.util.EntitiesCount; import org.openmetadata.schema.util.ServicesCount; +import org.openmetadata.sdk.PipelineServiceClient; import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.CustomExceptionMessage; import org.openmetadata.service.fernet.Fernet; import org.openmetadata.service.jdbi3.CollectionDAO.SystemDAO; +import org.openmetadata.service.migration.MigrationValidationClient; import org.openmetadata.service.resources.settings.SettingsCache; +import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.security.JwtFilter; import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.OpenMetadataConnectionBuilder; import org.openmetadata.service.util.RestUtil; import org.openmetadata.service.util.ResultList; @@ -32,10 +42,28 @@ public class SystemRepository { private static final String FAILED_TO_UPDATE_SETTINGS = "Failed to Update Settings"; public static final String INTERNAL_SERVER_ERROR_WITH_REASON = "Internal Server Error. Reason :"; private final SystemDAO dao; + private final MigrationValidationClient migrationValidationClient; + + private enum ValidationStepDescription { + DATABASE("Validate that we can properly run a query against the configured database."), + SEARCH("Validate that the search client is available."), + PIPELINE_SERVICE_CLIENT("Validate that the pipeline service client is available."), + JWT_TOKEN("Validate that the ingestion-bot JWT token can be properly decoded."), + MIGRATION("Validate that all the necessary migrations have been properly executed."); + + public final String key; + + ValidationStepDescription(String param) { + this.key = param; + } + } + + private static final String INDEX_NAME = "table_search_index"; public SystemRepository() { this.dao = Entity.getCollectionDAO().systemDAO(); Entity.setSystemRepository(this); + migrationValidationClient = MigrationValidationClient.getInstance(); } public EntitiesCount getAllEntitiesCount(ListFilter filter) { @@ -210,4 +238,98 @@ public class SystemRepository { } return JsonUtils.readValue(encryptedSetting, SlackAppConfiguration.class); } + + public ValidationResponse validateSystem( + OpenMetadataApplicationConfig applicationConfig, + PipelineServiceClient pipelineServiceClient, + JwtFilter jwtFilter) { + ValidationResponse validation = new ValidationResponse(); + + validation.setDatabase(getDatabaseValidation()); + validation.setSearchInstance(getSearchValidation()); + validation.setPipelineServiceClient(getPipelineServiceClientValidation(pipelineServiceClient)); + validation.setJwks(getJWKsValidation(applicationConfig, jwtFilter)); + validation.setMigrations(getMigrationValidation(migrationValidationClient)); + + return validation; + } + + private StepValidation getDatabaseValidation() { + try { + dao.testConnection(); + return new StepValidation() + .withDescription(ValidationStepDescription.DATABASE.key) + .withPassed(Boolean.TRUE); + } catch (Exception exc) { + return new StepValidation() + .withDescription(ValidationStepDescription.DATABASE.key) + .withPassed(Boolean.FALSE) + .withMessage(exc.getMessage()); + } + } + + private StepValidation getSearchValidation() { + SearchRepository searchRepository = Entity.getSearchRepository(); + if (Boolean.TRUE.equals(searchRepository.getSearchClient().isClientAvailable()) + && searchRepository.getSearchClient().indexExists(INDEX_NAME)) { + return new StepValidation() + .withDescription(ValidationStepDescription.SEARCH.key) + .withPassed(Boolean.TRUE); + } else { + return new StepValidation() + .withDescription(ValidationStepDescription.SEARCH.key) + .withPassed(Boolean.FALSE) + .withMessage("Search instance is not reachable or available"); + } + } + + private StepValidation getPipelineServiceClientValidation( + PipelineServiceClient pipelineServiceClient) { + PipelineServiceClientResponse pipelineResponse = pipelineServiceClient.getServiceStatus(); + if (pipelineResponse.getCode() == 200) { + return new StepValidation() + .withDescription(ValidationStepDescription.PIPELINE_SERVICE_CLIENT.key) + .withPassed(Boolean.TRUE); + } else { + return new StepValidation() + .withDescription(ValidationStepDescription.PIPELINE_SERVICE_CLIENT.key) + .withPassed(Boolean.FALSE) + .withMessage(pipelineResponse.getReason()); + } + } + + private StepValidation getJWKsValidation( + OpenMetadataApplicationConfig applicationConfig, JwtFilter jwtFilter) { + OpenMetadataConnection openMetadataServerConnection = + new OpenMetadataConnectionBuilder(applicationConfig).build(); + try { + jwtFilter.validateAndReturnDecodedJwtToken( + openMetadataServerConnection.getSecurityConfig().getJwtToken()); + return new StepValidation() + .withDescription(ValidationStepDescription.JWT_TOKEN.key) + .withPassed(Boolean.TRUE); + } catch (Exception e) { + return new StepValidation() + .withDescription(ValidationStepDescription.JWT_TOKEN.key) + .withPassed(Boolean.FALSE) + .withMessage(e.getMessage()); + } + } + + private StepValidation getMigrationValidation( + MigrationValidationClient migrationValidationClient) { + List currentVersions = migrationValidationClient.getCurrentVersions(); + if (currentVersions.equals(migrationValidationClient.getExpectedMigrationList())) { + return new StepValidation() + .withDescription(ValidationStepDescription.MIGRATION.key) + .withPassed(Boolean.TRUE); + } + return new StepValidation() + .withDescription(ValidationStepDescription.MIGRATION.key) + .withPassed(Boolean.FALSE) + .withMessage( + String.format( + "Found the versions [%s], but expected [%s]", + currentVersions, migrationValidationClient.getExpectedMigrationList())); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClient.java new file mode 100644 index 00000000000..d803ebc7dd9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClient.java @@ -0,0 +1,72 @@ +package org.openmetadata.service.migration; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.jdbi3.MigrationDAO; + +@Slf4j +public class MigrationValidationClient { + @Getter public static MigrationValidationClient instance; + + private final MigrationDAO migrationDAO; + private final OpenMetadataApplicationConfig config; + @Getter private final List expectedMigrationList; + + private MigrationValidationClient( + MigrationDAO migrationDAO, OpenMetadataApplicationConfig config) { + this.migrationDAO = migrationDAO; + this.config = config; + this.expectedMigrationList = loadExpectedMigrationList(); + } + + public static MigrationValidationClient initialize( + MigrationDAO migrationDAO, OpenMetadataApplicationConfig config) { + + if (instance == null) { + instance = new MigrationValidationClient(migrationDAO, config); + } + return instance; + } + + public List getCurrentVersions() { + return migrationDAO.getMigrationVersions(); + } + + private List loadExpectedMigrationList() { + try { + String nativePath = config.getMigrationConfiguration().getNativePath(); + String extensionPath = config.getMigrationConfiguration().getExtensionPath(); + + List availableOMNativeMigrations = getMigrationFilesFromPath(nativePath); + + // If we only have OM migrations, return them + if (extensionPath == null || extensionPath.isEmpty()) { + return availableOMNativeMigrations; + } + + // Otherwise, fetch the extension migration and sort the results + List availableOMExtensionMigrations = getMigrationFilesFromPath(extensionPath); + + return Stream.concat( + availableOMNativeMigrations.stream(), availableOMExtensionMigrations.stream()) + .sorted() + .toList(); + } catch (Exception e) { + LOG.error("Error loading expected migration list", e); + return List.of(); + } + } + + private List getMigrationFilesFromPath(String path) { + return Arrays.stream(Objects.requireNonNull(new File(path).listFiles(File::isDirectory))) + .map(File::getName) + .sorted() + .toList(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClientException.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClientException.java new file mode 100644 index 00000000000..2636d2839f8 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/MigrationValidationClientException.java @@ -0,0 +1,44 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.migration; + +import javax.ws.rs.core.Response; +import org.openmetadata.sdk.exception.WebServiceException; + +public class MigrationValidationClientException extends WebServiceException { + private static final String BY_NAME_MESSAGE = "Migration Validation Exception [%s] due to [%s]."; + private static final String ERROR_TYPE = "MIGRATION_VALIDATION"; + + public MigrationValidationClientException(String message) { + super(Response.Status.BAD_REQUEST, ERROR_TYPE, message); + } + + private MigrationValidationClientException(Response.Status status, String message) { + super(status, ERROR_TYPE, message); + } + + public static MigrationValidationClientException byMessage( + String name, String errorMessage, Response.Status status) { + return new MigrationValidationClientException(status, buildMessageByName(name, errorMessage)); + } + + public static MigrationValidationClientException byMessage(String name, String errorMessage) { + return new MigrationValidationClientException( + Response.Status.BAD_REQUEST, buildMessageByName(name, errorMessage)); + } + + private static String buildMessageByName(String name, String errorMessage) { + return String.format(BY_NAME_MESSAGE, name, errorMessage); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java index 319f0e659a4..79e9a76fbb7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java @@ -152,12 +152,12 @@ public class ConfigResource { @GET @Path(("/pipeline-service-client")) @Operation( - operationId = "getAirflowConfiguration", - summary = "Get airflow configuration", + operationId = "getPipelineServiceConfiguration", + summary = "Get Pipeline Service Client configuration", responses = { @ApiResponse( responseCode = "200", - description = "Airflow configuration", + description = "Pipeline Service Client configuration", content = @Content( mediaType = "application/json", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java index 93cfd5d89b5..d7193b16d0c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java @@ -34,16 +34,20 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.auth.EmailRequest; import org.openmetadata.schema.settings.Settings; import org.openmetadata.schema.settings.SettingsType; +import org.openmetadata.schema.system.ValidationResponse; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.util.EntitiesCount; import org.openmetadata.schema.util.ServicesCount; +import org.openmetadata.sdk.PipelineServiceClient; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; import org.openmetadata.service.exception.UnhandledServerException; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.SystemRepository; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.JwtFilter; import org.openmetadata.service.util.EmailUtil; import org.openmetadata.service.util.ResultList; @@ -55,19 +59,26 @@ import org.openmetadata.service.util.ResultList; @Collection(name = "system") @Slf4j public class SystemResource { - public static final String COLLECTION_PATH = "/v1/util"; + public static final String COLLECTION_PATH = "/v1/system"; private final SystemRepository systemRepository; private final Authorizer authorizer; private OpenMetadataApplicationConfig applicationConfig; + private PipelineServiceClient pipelineServiceClient; + private JwtFilter jwtFilter; public SystemResource(Authorizer authorizer) { this.systemRepository = Entity.getSystemRepository(); this.authorizer = authorizer; } - @SuppressWarnings("unused") // Method used for reflection public void initialize(OpenMetadataApplicationConfig config) { this.applicationConfig = config; + this.pipelineServiceClient = + PipelineServiceClientFactory.createPipelineServiceClient( + config.getPipelineServiceClientConfiguration()); + + this.jwtFilter = + new JwtFilter(config.getAuthenticationConfiguration(), config.getAuthorizerConfiguration()); } public static class SettingsList extends ResultList { @@ -287,4 +298,24 @@ public class SystemResource { ListFilter filter = new ListFilter(include); return systemRepository.getAllServicesCount(filter); } + + @GET + @Path("/status") + @Operation( + operationId = "validateDeployment", + summary = "Validate the OpenMetadata deployment", + description = + "Check connectivity against your database, elasticsearch/opensearch, migrations,...", + responses = { + @ApiResponse( + responseCode = "200", + description = "validation OK", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = ServicesCount.class))) + }) + public ValidationResponse validate() { + return systemRepository.validateSystem(applicationConfig, pipelineServiceClient, jwtFilter); + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java index 04ed3eee26e..e6217dfc920 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java @@ -40,6 +40,7 @@ import org.openmetadata.schema.entity.teams.AuthenticationMechanism; import org.openmetadata.schema.security.client.GoogleSSOClientConfig; import org.openmetadata.schema.settings.Settings; import org.openmetadata.schema.settings.SettingsType; +import org.openmetadata.schema.system.ValidationResponse; import org.openmetadata.schema.util.EntitiesCount; import org.openmetadata.schema.util.ServicesCount; import org.openmetadata.service.OpenMetadataApplicationConfig; @@ -301,6 +302,19 @@ public class SystemResourceTest extends OpenMetadataApplicationTest { Assertions.assertEquals(beforeUserCount, afterUserCount); } + @Test + void validate_test() throws HttpResponseException { + ValidationResponse response = getValidation(); + + // Check migrations are OK + Assertions.assertEquals(Boolean.TRUE, response.getMigrations().getPassed()); + } + + private static ValidationResponse getValidation() throws HttpResponseException { + WebTarget target = getResource("system/status"); + return TestUtils.get(target, ValidationResponse.class, ADMIN_AUTH_HEADERS); + } + private static EntitiesCount getEntitiesCount() throws HttpResponseException { WebTarget target = getResource("system/entities/count"); return TestUtils.get(target, EntitiesCount.class, ADMIN_AUTH_HEADERS); diff --git a/openmetadata-spec/src/main/resources/json/schema/system/validationResponse.json b/openmetadata-spec/src/main/resources/json/schema/system/validationResponse.json new file mode 100644 index 00000000000..d3749bfc65c --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/system/validationResponse.json @@ -0,0 +1,53 @@ +{ + "$id": "https://open-metadata.org/schema/system/validationResponse.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SystemValidationResponse", + "description": "Define the system validation response", + "type": "object", + "javaType": "org.openmetadata.schema.system.ValidationResponse", + "definitions": { + "stepValidation": { + "javaType": "org.openmetadata.schema.system.StepValidation", + "type": "object", + "properties": { + "description": { + "description": "Validation description. What is being tested?", + "type": "string" + }, + "passed": { + "description": "Did the step validation successfully?", + "type": "boolean" + }, + "message": { + "description": "Results or exceptions to be shared after running the test.", + "type": "string", + "default": null + } + }, + "additionalProperties": false + } + }, + "properties": { + "database": { + "description": "Database connectivity check", + "$ref": "#/definitions/stepValidation" + }, + "searchInstance": { + "description": "Search instance connectivity check", + "$ref": "#/definitions/stepValidation" + }, + "pipelineServiceClient": { + "description": "Pipeline Service Client connectivity check", + "$ref": "#/definitions/stepValidation" + }, + "jwks": { + "description": "JWKs validation", + "$ref": "#/definitions/stepValidation" + }, + "migrations": { + "description": "List migration results", + "$ref": "#/definitions/stepValidation" + } + }, + "additionalProperties": false +} \ No newline at end of file