Fix: Superset encryption and decryption not working as expected (#9827)

* Fix superset sample data

* Update ingestion/examples/superset_data/service.json

* Update ingestion/examples/superset_data/service.json

* fix: pytest error (#9824)

* fix: pytest error

* fix: linting

* increased verbosity

* empty commit to re-run tests

* print registry and test definition set

* renamed columnValuesToBeUnique fqn

* removed print statements + verbosity

* Docs Using a Custom Ingestion Image in helm (#9835)

* Docs Using a Custom Ingestion Image in helm

* Update openmetadata-docs/content/deployment/kubernetes/index.md

Co-authored-by: Akash Jain <15995028+akash-jain-10@users.noreply.github.com>

* Update openmetadata-docs/content/deployment/kubernetes/index.md

Co-authored-by: Akash Jain <15995028+akash-jain-10@users.noreply.github.com>

* Update openmetadata-docs/content/deployment/kubernetes/index.md

* Update index.md

Co-authored-by: Akash Jain <15995028+akash-jain-10@users.noreply.github.com>

* fix(ui): ui changes for #9695 (#9770)

* Fix #9695: Teams show up in lower case

* changed owner advanced search filter field from name to display name

* fixed minor quick filter bug

* fixed failing cypress tests

* fixed failing unit tests

* Fixed failing cypress tests

* added comments in unit tests for better understanding of changes

* fixed failing cypress tests for advanced search

* improved logic for showing advanced search options

Co-authored-by: Sriharsha Chintalapani <harsha@getcollate.io>

* Added powerbi docs for new field (#9836)

* Fix#9460: Avoid reuse inspector to get view definition (#9821)

* Avoid reuse inspector to get view definition

* Update openmetadata-sqllineage version

* Feat: Update GH action for E2E tests (#9839)

* Update E2E CI tests GH action

* Address PR comments

* Add converter for services when encypting or decrypting services objects

* Update E2E CI tests GH action

Co-authored-by: Teddy <teddy.crepineau@gmail.com>
Co-authored-by: Pere Miquel Brull <peremiquelbrull@gmail.com>
Co-authored-by: Akash Jain <15995028+akash-jain-10@users.noreply.github.com>
Co-authored-by: Aniket Katkar <aniketkatkar97@gmail.com>
Co-authored-by: Sriharsha Chintalapani <harsha@getcollate.io>
Co-authored-by: Onkar Ravgan <onkar.10r@gmail.com>

.github/workflows/py-cli-e2e-tests.yml

@@ -101,3 +101,8 @@ jobs:
       env:
         SLACK_WEBHOOK_URL: ${{ secrets.E2E_SLACK_WEBHOOK }}
         SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+
+    - name: Force failure
+      if: steps.e2e-test.outcome != 'success'
+      run: |
+        exit 1

ingestion/examples/superset_data/service.json

@@ -3,6 +3,9 @@
 	"serviceType": "Superset",
 	"description": "Supset Service",
   	"dashboardUrl": "http://localhost:8088",
-  	"username": "admin",
-  	"password": "admin"
+	"connection": {
+		"provider": "db",
+		"username": "admin",
+		"password": "admin"
+	}
 }

openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManager.java

@@ -31,9 +31,9 @@ import org.openmetadata.schema.security.secrets.SecretsManagerProvider;
 import org.openmetadata.service.exception.InvalidServiceConnectionException;
 import org.openmetadata.service.exception.SecretsManagerException;
 import org.openmetadata.service.fernet.Fernet;
+import org.openmetadata.service.secrets.converter.service.ServiceConverterFactory;
 import org.openmetadata.service.util.AuthenticationMechanismBuilder;
 import org.openmetadata.service.util.IngestionPipelineBuilder;
-import org.openmetadata.service.util.JsonUtils;
 
 public abstract class SecretsManager {
   @Getter private final String clusterPrefix;
@@ -52,7 +52,7 @@ public abstract class SecretsManager {
       Object connectionConfig, String connectionType, String connectionName, ServiceType serviceType, boolean encrypt) {
     try {
       Class<?> clazz = createConnectionConfigClass(connectionType, extractConnectionPackageName(serviceType));
-      Object newConnectionConfig = JsonUtils.convertValue(connectionConfig, clazz);
+      Object newConnectionConfig = ServiceConverterFactory.getConverter(clazz).convertFromJson(connectionConfig);
       return encryptOrDecryptPasswordFields(
           newConnectionConfig, buildSecretId(true, serviceType.value(), connectionName), encrypt);
     } catch (Exception e) {

openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/DefaultServiceConverter.java

@@ -0,0 +1,21 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.openmetadata.service.secrets.converter.service;
+
+public class DefaultServiceConverter extends ServiceConverter {
+
+  public DefaultServiceConverter(Class<?> serviceClass) {
+    super(serviceClass);
+  }
+}

openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/ServiceConverter.java

@@ -0,0 +1,36 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.openmetadata.service.secrets.converter.service;
+
+import org.openmetadata.service.util.JsonUtils;
+
+/**
+ * Currently when an object is converted into a specific class using `JsonUtils.convertValue` there`Object` fields that
+ * are not converted into any concrete class which could lead to assign a `LinkedMap` to the `Object` field.
+ *
+ * <p>This abstract class wrap these `JsonUtils.convertValue` adding transformation to those `Object` fields into
+ * specific classes.
+ */
+public abstract class ServiceConverter {
+
+  protected Class<?> serviceClass;
+
+  public ServiceConverter(Class<?> serviceClass) {
+    this.serviceClass = serviceClass;
+  }
+
+  public Object convertFromJson(Object connectionConfig) {
+    return JsonUtils.convertValue(connectionConfig, this.serviceClass);
+  }
+}

openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/ServiceConverterFactory.java

@@ -0,0 +1,31 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.openmetadata.service.secrets.converter.service;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.openmetadata.schema.services.connections.dashboard.SupersetConnection;
+
+public class ServiceConverterFactory {
+
+  private static final Map<Class<?>, ServiceConverter> converterMap = new HashMap<>();
+
+  static {
+    converterMap.put(SupersetConnection.class, new SupersetServiceConverter(SupersetConnection.class));
+  }
+
+  public static ServiceConverter getConverter(Class<?> serviceClass) {
+    return converterMap.getOrDefault(serviceClass, new DefaultServiceConverter(serviceClass));
+  }
+}

openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/service/SupersetServiceConverter.java

@@ -0,0 +1,59 @@
+/*
+ *  Copyright 2021 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.openmetadata.service.secrets.converter.service;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import org.openmetadata.schema.entity.utils.SupersetApiConnection;
+import org.openmetadata.schema.services.connections.dashboard.SupersetConnection;
+import org.openmetadata.schema.services.connections.database.MysqlConnection;
+import org.openmetadata.schema.services.connections.database.PostgresConnection;
+import org.openmetadata.service.util.JsonUtils;
+
+/** Factory class to get a `ServiceConverter` based on the service class. */
+public class SupersetServiceConverter extends ServiceConverter {
+
+  private static final List<Class<?>> CONNECTION_CLASSES =
+      List.of(SupersetApiConnection.class, MysqlConnection.class, PostgresConnection.class);
+
+  public SupersetServiceConverter(Class<?> serviceClass) {
+    super(serviceClass);
+  }
+
+  @Override
+  public Object convertFromJson(Object connectionConfig) {
+    SupersetConnection supersetConnection =
+        (SupersetConnection) JsonUtils.convertValue(connectionConfig, this.serviceClass);
+    if (supersetConnection.getConnection() instanceof Map) {
+      Object connection =
+          CONNECTION_CLASSES.stream()
+              .map(clazz -> convertConnectionFromJson(supersetConnection.getConnection(), clazz))
+              .filter(Objects::nonNull)
+              .findFirst()
+              .orElse(null);
+      supersetConnection.setConnection(connection);
+    }
+    return supersetConnection;
+  }
+
+  private Object convertConnectionFromJson(Object connection, Class<?> clazz) {
+    try {
+      return JsonUtils.convertValue(connection, clazz);
+    } catch (Exception ignore) {
+      // this can be ignored
+      return null;
+    }
+  }
+}