MNIOR: feat(apps): support config file (#17872)

* feat(apps): support config file

- added support for app config files
- removed AppPrivateConfig from the OpenMetadata server configuration

* use dorpwizard utility classes for resolving environment variables in the config

* moved fields to class level

* format

openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java

@@ -23,7 +23,6 @@ import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
 import lombok.Getter;
 import lombok.Setter;
-import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration;
 import org.openmetadata.schema.api.configuration.dataQuality.DataQualityConfiguration;
 import org.openmetadata.schema.api.configuration.events.EventHandlerConfiguration;
 import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration;
@@ -114,9 +113,6 @@ public class OpenMetadataApplicationConfig extends Configuration {
   @JsonProperty("dataQualityConfiguration")
   private DataQualityConfiguration dataQualityConfiguration;
 
-  @JsonProperty("applications")
-  private AppsPrivateConfiguration appsPrivateConfiguration;
-
   @JsonProperty("limits")
   private LimitsConfiguration limitsConfiguration;
 

openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java

@@ -1,17 +1,17 @@
 package org.openmetadata.service.apps;
 
-import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
 import static org.openmetadata.service.apps.scheduler.AppScheduler.APPS_JOB_GROUP;
 import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY;
 import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_NAME;
 
+import io.dropwizard.configuration.ConfigurationException;
+import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.Collection;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig;
-import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration;
 import org.openmetadata.schema.entity.app.App;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
 import org.openmetadata.service.apps.scheduler.AppScheduler;
@@ -33,12 +33,11 @@ public class ApplicationHandler {
 
   @Getter private static ApplicationHandler instance;
   private final OpenMetadataApplicationConfig config;
-  private final AppsPrivateConfiguration privateConfiguration;
   private final AppRepository appRepository;
+  private final ConfigurationReader configReader = new ConfigurationReader();
 
   private ApplicationHandler(OpenMetadataApplicationConfig config) {
     this.config = config;
-    this.privateConfiguration = config.getAppsPrivateConfiguration();
     this.appRepository = new AppRepository();
   }
 
@@ -55,28 +54,28 @@ public class ApplicationHandler {
   public void setAppRuntimeProperties(App app) {
     app.setOpenMetadataServerConnection(
         new OpenMetadataConnectionBuilder(config, app.getBot().getName()).build());
-
+    try {
-    if (privateConfiguration != null
+      AppPrivateConfig appPrivateConfig = configReader.readConfigFromResource(app.getName());
-        && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) {
+      app.setPreview(appPrivateConfig.getPreview());
-      for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) {
+      app.setPrivateConfiguration(appPrivateConfig.getParameters());
-        if (app.getName().equals(appPrivateConfig.getName())) {
+    } catch (IOException e) {
-          app.setPreview(appPrivateConfig.getPreview());
+      LOG.debug("Config file for app {} not found: ", app.getName(), e);
-          app.setPrivateConfiguration(appPrivateConfig.getParameters());
+    } catch (ConfigurationException e) {
-        }
+      LOG.error("Error reading config file for app {}", app.getName(), e);
-      }
     }
   }
 
   public Boolean isPreview(String appName) {
-    if (privateConfiguration != null
+    try {
-        && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) {
+      AppPrivateConfig appPrivateConfig = configReader.readConfigFromResource(appName);
-      for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) {
+      return appPrivateConfig.getPreview();
-        if (appName.equals(appPrivateConfig.getName())) {
+    } catch (IOException e) {
-          return appPrivateConfig.getPreview();
+      LOG.debug("Config file for app {} not found: ", appName, e);
-        }
+      return false;
-      }
+    } catch (ConfigurationException e) {
+      LOG.error("Error reading config file for app {}", appName, e);
+      return false;
     }
-    return false;
   }
 
   public void triggerApplicationOnDemand(

openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java

@@ -0,0 +1,57 @@
+package org.openmetadata.service.apps;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import io.dropwizard.configuration.ConfigurationException;
+import io.dropwizard.configuration.EnvironmentVariableSubstitutor;
+import io.dropwizard.configuration.FileConfigurationSourceProvider;
+import io.dropwizard.configuration.SubstitutingSourceProvider;
+import io.dropwizard.configuration.YamlConfigurationFactory;
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Map;
+import org.apache.commons.text.StringSubstitutor;
+import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig;
+import org.openmetadata.service.util.JsonUtils;
+
+public class ConfigurationReader {
+  private final StringSubstitutor substitutor;
+  private final ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+  private final YamlConfigurationFactory<Object> factory =
+      new YamlConfigurationFactory<>(Object.class, null, mapper, "dw");
+
+  public ConfigurationReader(Map<String, String> envMap) {
+    // envMap is for custom environment variables (e.g., for testing), defaulting to the system
+    // environment.
+    substitutor =
+        envMap == null ? new EnvironmentVariableSubstitutor(false) : new StringSubstitutor(envMap);
+  }
+
+  public ConfigurationReader() {
+    this(System.getenv());
+  }
+
+  public AppPrivateConfig readConfigFromResource(String appName)
+      throws IOException, ConfigurationException {
+    String configFilePath = "applications/" + appName + "/config.yaml";
+    URL resource = ConfigurationReader.class.getClassLoader().getResource(configFilePath);
+    if (resource == null) {
+      throw new IOException("Configuration file not found: " + configFilePath);
+    }
+    File configFile = new File(resource.getFile());
+    return JsonUtils.convertValue(readConfigFile(configFile), AppPrivateConfig.class);
+  }
+
+  public Map<String, Object> readConfigFile(File configFile)
+      throws IOException, ConfigurationException {
+    try {
+      return (Map<String, Object>)
+          factory.build(
+              new SubstitutingSourceProvider(new FileConfigurationSourceProvider(), substitutor),
+              configFile.getAbsolutePath());
+    } catch (ClassCastException e) {
+      throw new RuntimeException("Configuration file is not a valid YAML file", e);
+    }
+  }
+}

openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/ConfigurationReaderTest.java

@@ -0,0 +1,57 @@
+package org.openmetadata.service.resources.apps;
+
+import static org.junit.Assert.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import io.dropwizard.configuration.ConfigurationException;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig;
+import org.openmetadata.service.apps.ConfigurationReader;
+
+public class ConfigurationReaderTest {
+
+  @Test
+  public void testReadConfigFile() throws IOException, ConfigurationException {
+    ConfigurationReader reader =
+        new ConfigurationReader(
+            Map.of(
+                "ENV_VAR",
+                "resolvedValue",
+                "NESTED_ENV_VAR",
+                "nestedValue",
+                "LIST_ENV_VAR",
+                "value1"));
+    AppPrivateConfig appConfig = reader.readConfigFromResource("TestApplication");
+    assertNotNull(appConfig);
+    assertEquals("value1", appConfig.getParameters().getAdditionalProperties().get("key1"));
+    assertEquals("resolvedValue", appConfig.getParameters().getAdditionalProperties().get("key2"));
+    assertEquals("", appConfig.getParameters().getAdditionalProperties().get("emptyKey"));
+    assertEquals("default", appConfig.getParameters().getAdditionalProperties().get("defaultKey"));
+    Map<String, String> nested =
+        (Map<String, String>) appConfig.getParameters().getAdditionalProperties().get("nested");
+    assertEquals("nestedValue", nested.get("nestedKey"));
+    List<String> list =
+        (List<String>) appConfig.getParameters().getAdditionalProperties().get("list");
+    assertEquals("value1", list.get(1));
+  }
+
+  @Test
+  public void testInvalidConfig() {
+    ConfigurationReader reader = new ConfigurationReader();
+    assertThrows(RuntimeException.class, () -> reader.readConfigFromResource("InvalidConfig"));
+  }
+
+  @Test
+  public void missingConfig() {
+    ConfigurationReader reader = new ConfigurationReader();
+    assertThrows(
+        IOException.class,
+        () -> {
+          reader.readConfigFromResource("missing");
+        });
+  }
+}

openmetadata-service/src/test/resources/applications/InvalidConfig/config.yaml

@@ -0,0 +1,3 @@
+---
+- a
+- b

openmetadata-service/src/test/resources/applications/TestApplication/config.yaml

@@ -0,0 +1,10 @@
+parameters:
+  key1: value1
+  key2: ${ENV_VAR}
+  emptyKey: ${UNDEFINED_ENV_VAR:-""}
+  defaultKey: ${UNDEFINED_ENV_VAR:-default}
+  nested:
+    nestedKey: ${NESTED_ENV_VAR}
+  list:
+    - elem1
+    - ${LIST_ENV_VAR}