diff --git a/.github/workflows/py-cli-e2e-tests.yml b/.github/workflows/py-cli-e2e-tests.yml index 4f2c006dd48..56c9225b8d9 100644 --- a/.github/workflows/py-cli-e2e-tests.yml +++ b/.github/workflows/py-cli-e2e-tests.yml @@ -101,3 +101,8 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.E2E_SLACK_WEBHOOK }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + + - name: Force failure + if: steps.e2e-test.outcome != 'success' + run: | + exit 1 diff --git a/ingestion/examples/superset_data/service.json b/ingestion/examples/superset_data/service.json index 81a97872e60..cef465e9959 100644 --- a/ingestion/examples/superset_data/service.json +++ b/ingestion/examples/superset_data/service.json @@ -3,6 +3,9 @@ "serviceType": "Superset", "description": "Supset Service", "dashboardUrl": "http://localhost:8088", - "username": "admin", - "password": "admin" + "connection": { + "provider": "db", + "username": "admin", + "password": "admin" + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java index 93b29c9a320..d4171368304 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java @@ -31,9 +31,9 @@ import org.openmetadata.schema.security.secrets.SecretsManagerProvider; import org.openmetadata.service.exception.InvalidServiceConnectionException; import org.openmetadata.service.exception.SecretsManagerException; import org.openmetadata.service.fernet.Fernet; +import org.openmetadata.service.secrets.converter.service.ServiceConverterFactory; import org.openmetadata.service.util.AuthenticationMechanismBuilder; import org.openmetadata.service.util.IngestionPipelineBuilder; -import org.openmetadata.service.util.JsonUtils; public abstract class SecretsManager { @Getter private final String clusterPrefix; @@ -52,7 +52,7 @@ public abstract class SecretsManager { Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType, boolean encrypt) { try { Class clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType)); - Object newConnectionConfig = JsonUtils.convertValue(connectionConfig, clazz); + Object newConnectionConfig = ServiceConverterFactory.getConverter(clazz).convertFromJson(connectionConfig); return encryptOrDecryptPasswordFields( newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName), encrypt); } catch (Exception e) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/DefaultServiceConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/DefaultServiceConverter.java new file mode 100644 index 00000000000..7ecf2bea28c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/DefaultServiceConverter.java @@ -0,0 +1,21 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter.service; + +public class DefaultServiceConverter extends ServiceConverter { + + public DefaultServiceConverter(Class serviceClass) { + super(serviceClass); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/ServiceConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/ServiceConverter.java new file mode 100644 index 00000000000..b631325d9e9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/ServiceConverter.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter.service; + +import org.openmetadata.service.util.JsonUtils; + +/** + * Currently when an object is converted into a specific class using `JsonUtils.convertValue` there`Object` fields that + * are not converted into any concrete class which could lead to assign a `LinkedMap` to the `Object` field. + * + *

This abstract class wrap these `JsonUtils.convertValue` adding transformation to those `Object` fields into + * specific classes. + */ +public abstract class ServiceConverter { + + protected Class serviceClass; + + public ServiceConverter(Class serviceClass) { + this.serviceClass = serviceClass; + } + + public Object convertFromJson(Object connectionConfig) { + return JsonUtils.convertValue(connectionConfig, this.serviceClass); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/ServiceConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/ServiceConverterFactory.java new file mode 100644 index 00000000000..1a726423bf7 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/ServiceConverterFactory.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter.service; + +import java.util.HashMap; +import java.util.Map; +import org.openmetadata.schema.services.connections.dashboard.SupersetConnection; + +public class ServiceConverterFactory { + + private static final Map, ServiceConverter> converterMap = new HashMap<>(); + + static { + converterMap.put(SupersetConnection.class, new SupersetServiceConverter(SupersetConnection.class)); + } + + public static ServiceConverter getConverter(Class serviceClass) { + return converterMap.getOrDefault(serviceClass, new DefaultServiceConverter(serviceClass)); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/SupersetServiceConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/SupersetServiceConverter.java new file mode 100644 index 00000000000..d1a30472001 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/SupersetServiceConverter.java @@ -0,0 +1,59 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter.service; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.openmetadata.schema.entity.utils.SupersetApiConnection; +import org.openmetadata.schema.services.connections.dashboard.SupersetConnection; +import org.openmetadata.schema.services.connections.database.MysqlConnection; +import org.openmetadata.schema.services.connections.database.PostgresConnection; +import org.openmetadata.service.util.JsonUtils; + +/** Factory class to get a `ServiceConverter` based on the service class. */ +public class SupersetServiceConverter extends ServiceConverter { + + private static final List> CONNECTION_CLASSES = + List.of(SupersetApiConnection.class, MysqlConnection.class, PostgresConnection.class); + + public SupersetServiceConverter(Class serviceClass) { + super(serviceClass); + } + + @Override + public Object convertFromJson(Object connectionConfig) { + SupersetConnection supersetConnection = + (SupersetConnection) JsonUtils.convertValue(connectionConfig, this.serviceClass); + if (supersetConnection.getConnection() instanceof Map) { + Object connection = + CONNECTION_CLASSES.stream() + .map(clazz -> convertConnectionFromJson(supersetConnection.getConnection(), clazz)) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + supersetConnection.setConnection(connection); + } + return supersetConnection; + } + + private Object convertConnectionFromJson(Object connection, Class clazz) { + try { + return JsonUtils.convertValue(connection, clazz); + } catch (Exception ignore) { + // this can be ignored + return null; + } + } +}