Fix 20325: Trigger external apps with config (#20397)

* wip

* feat: trigger external apps with override config

- Added in openmetadata-airflow-apis functionality to trigger DAG with feature.
- Modified openmetadata-airflow-apis application runner to accept override config from params.
- Added overloaded runPipeline with `Map<String,Object> config` to allow triggering apps with configuration. We might want to expand this to all ingestion pipelines. For now its just for apps.
- Implemented an example external app that can be used to test functionality of external apps. The app can be enabled by setting the `ENABLE_APP_HelloPipelines=true` environment variable.

* fix class doc for application

* fixed README for airflow apis

* fixes

* set HelloPipelines to disabeld by default

* fixed basedpywright errros

* fixed app schema

* reduced airflow client runPipeline to an overload with null config
removed duplicate call to runPipeline in AppResource

* Update openmetadata-docs/content/v1.7.x-SNAPSHOT/developers/applications/index.md

Co-authored-by: Matias Puerta <matias@getcollate.io>

* deleted documentation file

---------

Co-authored-by: Matias Puerta <matias@getcollate.io>

docker/development/docker-compose.yml

@@ -527,6 +527,8 @@ services:
       # To integrate GCP
       AIRFLOW__OPENMETADATA_SECRETS_MANAGER__GCP_PROJECT_ID: ${OM_SM_PROJECT_ID:-""}
 
+      # Apps
+      ENABLE_APP_HelloPipelines: "true"
 
     entrypoint: /bin/bash
     command:

ingestion/pyproject.toml

@@ -281,3 +281,5 @@ reportDeprecated = false
 reportMissingTypeStubs = false
 reportAny = false
 reportExplicitAny = false
+# @override was only added in python 3.12: https://docs.python.org/3/library/typing.html#typing.override
+reportImplicitOverride = false

ingestion/src/metadata/applications/example.py

@@ -0,0 +1,74 @@
+#  Copyright 2025 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Example external application
+"""
+from time import sleep
+from typing import Any
+
+from metadata.generated.schema.entity.applications.configuration.internal.helloPipelinesConfiguration import (
+    HelloPipelinesAppConfiguration,
+)
+from metadata.generated.schema.metadataIngestion.application import (
+    OpenMetadataApplicationConfig,
+)
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.logger import app_logger
+from metadata.workflow.application import AppRunner, InvalidAppConfiguration
+
+logger = app_logger()
+
+
+class HelloPipelines(AppRunner):
+    """
+    Example external application that sleeps for a given time and then echoes a message.
+    You can execute it with `metadata app -c <path-to-yaml>`
+    with a YAML file like:
+
+    sourcePythonClass: metadata.applications.example.HelloPipelines
+    appConfig:
+      type: HelloPipelines
+      sleep: 5
+      echo: this will be echoed
+    workflowConfig:
+      loggerLevel: INFO
+      openMetadataServerConfig:
+        hostPort: http://localhost:8585/api
+        authProvider: openmetadata
+        securityConfig:
+          jwtToken: "..."
+    """
+
+    def __init__(
+        self, config: OpenMetadataApplicationConfig, metadata: OpenMetadata[Any, Any]
+    ):
+        super().__init__(config, metadata)  # pyright: ignore [reportUnknownMemberType]
+        try:
+            self.app_config: HelloPipelinesAppConfiguration = (
+                HelloPipelinesAppConfiguration.model_validate(self.app_config)
+            )
+        except Exception as e:
+            raise InvalidAppConfiguration(
+                f"Hello pipelines received invalid configuration: {e}"
+            )
+
+    @property
+    def name(self) -> str:
+        return "HelloPipelines"
+
+    def run(self) -> None:
+        logger.info(f"sleeping for {self.app_config.sleep}")
+        sleep(self.app_config.sleep)
+        logger.info("echoing")
+        logger.info(self.app_config.echo)
+
+    def close(self) -> None:
+        """Nothing to close"""

openmetadata-airflow-apis/README.md

@@ -5,11 +5,15 @@ OpenMetadata workflow definition and manage DAGS and tasks.
 
 ## Development
 
-You can run `make branch=issue-3659-v2 test_up` and specify any branch from OpenMetadata that you'd
-need to test the changes in the APIs. This will prepare a separated airflow container.
+The file [`development/airflow/airflow.cfg`](./development/airflow/airflow.cfg) contains configuration which runs based on
+the airflow server deployed by the quick-start and development compose files.
 
-The command will build the image by downloading the branch changes inside the container. This helps us
-test the REST APIs using some ongoing changes on OpenMetadata as well.
+You ca run the following command to start the development environment:
+
+```bash
+export AIRFLOW_HOME=$(pwd)/openmetadata-airflow-managed-api/development/airflow
+airflow webserver
+```
 
 ## Requirements
 

openmetadata-airflow-apis/development/airflow/.gitignore

@@ -0,0 +1,5 @@
+logs
+dags
+dag_generated_configs
+airflow-webserver.pid
+webserver_config.py

openmetadata-airflow-apis/development/airflow/airflow.cfg

@@ -0,0 +1,40 @@
+[database]
+# The SQLAlchemy connection string to the metadata database.
+sql_alchemy_conn = mysql+mysqldb://airflow_user:airflow_pass@127.0.0.1:3306/airflow_db
+sql_engine_encoding = utf-8
+sql_alchemy_pool_enabled = True
+sql_alchemy_pool_size = 5
+sql_alchemy_max_overflow = 10
+sql_alchemy_pool_recycle = 1800
+sql_alchemy_pool_pre_ping = True
+
+[api]
+enable_experimental_api = True
+access_control_allow_headers = *
+access_control_allow_methods = *
+access_control_allow_origins = *
+auth_backends = airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session
+
+[webserver]
+web_server_host = 0.0.0.0
+web_server_port = 8080
+expose_config = True
+expose_hostname = True
+expose_stacktrace = True
+workers = 1
+threaded = True
+
+[core]
+dags_are_paused_at_creation = False
+load_examples = False
+executor = LocalExecutor
+parallelism = 1
+max_active_tasks_per_dag = 1
+max_active_runs_per_dag = 1
+
+[logging]
+logging_level = DEBUG
+fab_logging_level = DEBUG
+
+[openmetadata_airflow_apis]
+dag_generated_configs = ${AIRFLOW_HOME}/dag_generated_configs

openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/trigger.py

@@ -16,7 +16,11 @@ from typing import Callable
 
 from flask import Blueprint, Response, request
 from openmetadata_managed_apis.api.response import ApiResponse
-from openmetadata_managed_apis.api.utils import get_request_arg, get_request_dag_id
+from openmetadata_managed_apis.api.utils import (
+    get_request_arg,
+    get_request_conf,
+    get_request_dag_id,
+)
 from openmetadata_managed_apis.operations.trigger import trigger
 from openmetadata_managed_apis.utils.logger import routes_logger
 
@@ -41,13 +45,14 @@ def get_fn(blueprint: Blueprint) -> Callable:
     @security.requires_access([(permissions.ACTION_CAN_EDIT, permissions.RESOURCE_DAG)])
     def trigger_dag() -> Response:
         """
-        Trigger a dag run
+        Trigger a dag run with optional configuration
         """
         dag_id = get_request_dag_id()
 
         try:
             run_id = get_request_arg(request, "run_id", raise_missing=False)
-            response = trigger(dag_id, run_id)
+            conf = get_request_conf()
+            response = trigger(dag_id, run_id, conf=conf)
 
             return response
 

openmetadata-airflow-apis/openmetadata_managed_apis/api/utils.py

@@ -99,6 +99,16 @@ def get_request_dag_id() -> Optional[str]:
     return clean_dag_id(raw_dag_id)
 
 
+def get_request_conf() -> Optional[dict]:
+    """
+    Try to fetch the conf from the JSON request. Return None if no conf is provided.
+    """
+    try:
+        return request.get_json().get("conf")
+    except Exception:
+        return None
+
+
 def get_dagbag():
     """
     Load the dagbag from Airflow settings

openmetadata-airflow-apis/openmetadata_managed_apis/operations/deploy.py

@@ -97,6 +97,8 @@ class DagDeployer:
         Store the airflow pipeline config in a JSON file and
         return the path for the Jinja rendering.
         """
+        # Create directory if it doesn't exist
+        dag_config_file_path.parent.mkdir(parents=True, exist_ok=True)
 
         logger.info(f"Saving file to {dag_config_file_path}")
         with open(dag_config_file_path, "w") as outfile:

openmetadata-airflow-apis/openmetadata_managed_apis/operations/trigger.py

@@ -17,17 +17,20 @@ try:
     from airflow.api.common.trigger_dag import trigger_dag
 except ImportError:
     from airflow.api.common.experimental.trigger_dag import trigger_dag
+
 from airflow.utils import timezone
 from flask import Response
 from openmetadata_managed_apis.api.response import ApiResponse
 
 
-def trigger(dag_id: str, run_id: Optional[str]) -> Response:
+def trigger(
+    dag_id: str, run_id: Optional[str], conf: Optional[dict] = None
+) -> Response:
     dag_run = trigger_dag(
         dag_id=dag_id,
         run_id=run_id,
-        conf=None,
         execution_date=timezone.utcnow(),
+        conf=conf,
     )
     return ApiResponse.success(
         {"message": f"Workflow [{dag_id}] has been triggered {dag_run}"}

openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py

@@ -37,9 +37,7 @@ from metadata.generated.schema.metadataIngestion.applicationPipeline import (
 from metadata.workflow.application import ApplicationWorkflow
 
 
-def application_workflow(
-    workflow_config: OpenMetadataApplicationConfig,
-):
+def application_workflow(workflow_config: OpenMetadataApplicationConfig, **context):
     """
     Task that creates and runs the ingestion workflow.
 
@@ -51,9 +49,15 @@ def application_workflow(
 
     set_operator_logger(workflow_config)
 
+    # set overridden app config
     config = json.loads(
         workflow_config.model_dump_json(exclude_defaults=False, mask_secrets=False)
     )
+    params = context.get("params") or {}
+    config["appConfig"] = {
+        **(config.get("appConfig") or {}),
+        **(params.get("appConfigOverride") or {}),
+    }
     workflow = ApplicationWorkflow.create(config)
     execute_workflow(workflow, workflow_config)
 
@@ -100,6 +104,9 @@ def build_application_dag(ingestion_pipeline: IngestionPipeline) -> DAG:
         ingestion_pipeline=ingestion_pipeline,
         workflow_config=application_workflow_config,
         workflow_fn=application_workflow,
+        params={
+            "appConfigOverride": None  # Default to None, will be overridden by trigger conf
+        },
     )
 
     return dag

openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py

@@ -369,9 +369,16 @@ def build_dag(
     ingestion_pipeline: IngestionPipeline,
     workflow_config: Union[OpenMetadataWorkflowConfig, OpenMetadataApplicationConfig],
     workflow_fn: Callable,
+    params: Optional[dict] = None,
 ) -> DAG:
     """
     Build a simple metadata workflow DAG
+    :param task_name: Name of the task
+    :param ingestion_pipeline: Pipeline configs
+    :param workflow_config: Workflow configurations
+    :param workflow_fn: Function to be executed
+    :param params: Optional parameters to pass to the operator
+    :return: DAG
     """
 
     with DAG(**build_dag_configs(ingestion_pipeline)) as dag:
@@ -393,6 +400,7 @@ def build_dag(
             owner=ingestion_pipeline.owners.root[0].name
             if (ingestion_pipeline.owners and ingestion_pipeline.owners.root)
             else "openmetadata",
+            params=params,
         )
 
         return dag

openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/MeteredPipelineServiceClient.java

@@ -80,6 +80,15 @@ public class MeteredPipelineServiceClient implements PipelineServiceClientInterf
         RUN, () -> this.decoratedClient.runPipeline(ingestionPipeline, service));
   }
 
+  @Override
+  public PipelineServiceClientResponse runPipeline(
+      IngestionPipeline ingestionPipeline,
+      ServiceEntityInterface service,
+      Map<String, Object> config) {
+    return this.respondWithMetering(
+        RUN, () -> this.decoratedClient.runPipeline(ingestionPipeline, service, config));
+  }
+
   @Override
   public PipelineServiceClientResponse deletePipeline(IngestionPipeline ingestionPipeline) {
     return this.respondWithMetering(

openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/airflow/AirflowRESTClient.java

@@ -63,6 +63,8 @@ public class AirflowRESTClient extends PipelineServiceClient {
   protected final URL serviceURL;
   private static final List<String> API_ENDPOINT_SEGMENTS = List.of("api", "v1", "openmetadata");
   private static final String DAG_ID = "dag_id";
+  private static final String CONF = "conf";
+  private static final String APP_CONFIG_OVERRIDE = "appConfigOverride";
 
   public AirflowRESTClient(PipelineServiceClientConfiguration config) throws KeyStoreException {
 
@@ -172,12 +174,23 @@ public class AirflowRESTClient extends PipelineServiceClient {
   @Override
   public PipelineServiceClientResponse runPipeline(
       IngestionPipeline ingestionPipeline, ServiceEntityInterface service) {
+    return runPipeline(ingestionPipeline, service, null);
+  }
+
+  @Override
+  public PipelineServiceClientResponse runPipeline(
+      IngestionPipeline ingestionPipeline,
+      ServiceEntityInterface service,
+      Map<String, Object> config) {
     String pipelineName = ingestionPipeline.getName();
     HttpResponse<String> response;
     try {
       String triggerUrl = buildURI("trigger").build().toString();
       JSONObject requestPayload = new JSONObject();
       requestPayload.put(DAG_ID, pipelineName);
+      if (config != null) {
+        requestPayload.put(CONF, Map.of(APP_CONFIG_OVERRIDE, config));
+      }
       response = post(triggerUrl, requestPayload.toString());
       if (response.statusCode() == 200) {
         return getResponse(200, response.body());

openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java

@@ -5,6 +5,7 @@ import static org.openmetadata.service.jdbi3.EntityRepository.validateOwners;
 
 import java.util.List;
 import java.util.UUID;
+import javax.validation.ConstraintViolationException;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.app.App;
 import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition;
@@ -12,10 +13,12 @@ import org.openmetadata.schema.entity.app.CreateApp;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.service.Entity;
+import org.openmetadata.service.exception.BadRequestException;
 import org.openmetadata.service.jdbi3.AppMarketPlaceRepository;
 import org.openmetadata.service.jdbi3.AppRepository;
 import org.openmetadata.service.mapper.EntityMapper;
 import org.openmetadata.service.util.EntityUtil;
+import org.openmetadata.service.util.JsonUtils;
 
 public class AppMapper implements EntityMapper<App, CreateApp> {
   @Override
@@ -65,6 +68,11 @@ public class AppMapper implements EntityMapper<App, CreateApp> {
 
   private void validateAndAddBot(App app, String botName) {
     AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION);
+    try {
+      JsonUtils.validateJsonSchema(app, App.class);
+    } catch (ConstraintViolationException e) {
+      throw BadRequestException.of("Invalid App: " + e.getMessage());
+    }
     if (!CommonUtil.nullOrEmpty(botName)) {
       app.setBot(Entity.getEntityReferenceByName(BOT, botName, Include.NON_DELETED));
     } else {

openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java

@@ -1,5 +1,7 @@
 package org.openmetadata.service.resources.apps;
 
+import java.util.Objects;
+import javax.validation.ConstraintViolationException;
 import javax.ws.rs.BadRequestException;
 import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition;
 import org.openmetadata.schema.entity.app.AppType;
@@ -48,16 +50,21 @@ public class AppMarketPlaceMapper
   }
 
   private void validateApplication(AppMarketPlaceDefinition app) {
-    // Check if the className Exists in classPath
-    if (app.getAppType().equals(AppType.Internal)) {
-      // Check class name exists
-      try {
-        Class.forName(app.getClassName());
-      } catch (ClassNotFoundException e) {
-        throw new BadRequestException(
-            "Application Cannot be registered, because the classname cannot be found on the Classpath.");
-      }
-    } else {
+    try {
+      JsonUtils.validateJsonSchema(app, AppMarketPlaceDefinition.class);
+      Class.forName(
+          Objects.requireNonNull(
+              app.getClassName(), "AppMarketPlaceDefinition.className cannot be null"));
+    } catch (ClassNotFoundException e) {
+      throw new BadRequestException(
+          "Application Cannot be registered, because the Class cannot be found on the Classpath: "
+              + app.getEventSubscriptions());
+    } catch (ConstraintViolationException | NullPointerException e) {
+      throw new BadRequestException(
+          "Application Cannot be registered, because the AppMarketPlaceDefinition is not valid: "
+              + e.getMessage());
+    }
+    if (app.getAppType().equals(AppType.External)) {
       PipelineServiceClientResponse response = pipelineServiceClient.validateAppRegistration(app);
       if (response.getCode() != 200) {
         throw new BadRequestException(

openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java

@@ -1065,12 +1065,8 @@ public class AppResource extends EntityResource<App, AppRepository> {
         IngestionPipeline ingestionPipeline = getIngestionPipeline(uriInfo, securityContext, app);
         ServiceEntityInterface service =
             Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED);
-        if (configPayload != null) {
-          throw new BadRequestException(
-              "Overriding app config is not supported for external applications.");
-        }
         PipelineServiceClientResponse response =
-            pipelineServiceClient.runPipeline(ingestionPipeline, service);
+            pipelineServiceClient.runPipeline(ingestionPipeline, service, configPayload);
         return Response.status(response.getCode()).entity(response).build();
       }
     }

openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java

@@ -7,6 +7,10 @@ import static org.openmetadata.service.jdbi3.EntityRepository.getEntitiesFromSee
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import javax.validation.ConstraintViolationException;
+import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition;
 import org.openmetadata.schema.entity.app.CreateAppMarketPlaceDefinitionReq;
 import org.openmetadata.schema.entity.teams.Role;
@@ -20,6 +24,7 @@ import org.openmetadata.service.jdbi3.RoleRepository;
 import org.openmetadata.service.jdbi3.TeamRepository;
 import org.openmetadata.service.resources.apps.AppMarketPlaceMapper;
 
+@Slf4j
 public class AppMarketPlaceUtil {
   public static void createAppMarketPlaceDefinitions(
       AppMarketPlaceRepository appMarketRepository, AppMarketPlaceMapper mapper)
@@ -46,15 +51,35 @@ public class AppMarketPlaceUtil {
       teamRepository.initOrganization();
     }
 
-    List<CreateAppMarketPlaceDefinitionReq> createAppMarketPlaceDefinitionReqs =
-        getEntitiesFromSeedData(
+    getEntitiesFromSeedData(
             APPLICATION,
             String.format(".*json/data/%s/.*\\.json$", Entity.APP_MARKET_PLACE_DEF),
-            CreateAppMarketPlaceDefinitionReq.class);
-    for (CreateAppMarketPlaceDefinitionReq definitionReq : createAppMarketPlaceDefinitionReqs) {
-      AppMarketPlaceDefinition definition = mapper.createToEntity(definitionReq, ADMIN_USER_NAME);
-      appMarketRepository.setFullyQualifiedName(definition);
-      appMarketRepository.createOrUpdate(null, definition, ADMIN_USER_NAME);
-    }
+            CreateAppMarketPlaceDefinitionReq.class)
+        .stream()
+        .filter(
+            req ->
+                Optional.ofNullable(System.getenv("ENABLE_APP_" + req.getName()))
+                    .map(val -> Objects.equals(val, "true"))
+                    .orElse(req.getEnabled()))
+        .filter(
+            req -> {
+              try {
+                JsonUtils.validateJsonSchema(req, CreateAppMarketPlaceDefinitionReq.class);
+                return true;
+              } catch (ConstraintViolationException e) {
+                LOG.error(
+                    "Error validating {}: {}",
+                    CreateAppMarketPlaceDefinitionReq.class.getSimpleName(),
+                    req.getName(),
+                    e);
+                return false;
+              }
+            })
+        .forEach(
+            req -> {
+              AppMarketPlaceDefinition definition = mapper.createToEntity(req, ADMIN_USER_NAME);
+              appMarketRepository.setFullyQualifiedName(definition);
+              appMarketRepository.createOrUpdate(null, definition, ADMIN_USER_NAME);
+            });
   }
 }

openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/HelloPipelines.json

@@ -0,0 +1,25 @@
+{
+  "name": "HelloPipelines",
+  "displayName": "Hello Pipelines",
+  "description": "Example external application to demonstrate the use of pipelines.",
+  "features": "Test external application functionality",
+  "appType": "external",
+  "appScreenshots": ["DataInsightsPic1.png"],
+  "developer": "Collate Inc.",
+  "developerUrl": "https://www.getcollate.io",
+  "privacyPolicyUrl": "https://www.getcollate.io",
+  "supportEmail": "support@getcollate.io",
+  "scheduleType": "ScheduledOrManual",
+  "permission": "All",
+  "className": "org.openmetadata.service.apps.AbstractNativeApplication",
+  "sourcePythonClass": "metadata.applications.example.HelloPipelines",
+  "runtime": {
+    "enabled": "true"
+  },
+  "appConfiguration": {
+    "type": "HelloPipelines",
+    "sleep": 10,
+    "echo": "hello pipelines"
+  },
+  "enabled": false
+}

openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClientInterface.java

@@ -105,6 +105,16 @@ public interface PipelineServiceClientInterface {
   PipelineServiceClientResponse runPipeline(
       IngestionPipeline ingestionPipeline, ServiceEntityInterface service);
 
+  /* Deploy run the pipeline at the pipeline service with ad-hoc custom configuration.
+   * This might not be supported by some pipeline service clients.*/
+  default PipelineServiceClientResponse runPipeline(
+      IngestionPipeline ingestionPipeline,
+      ServiceEntityInterface service,
+      Map<String, Object> config) {
+    throw new UnsupportedOperationException(
+        "This operation is not supported by this pipeline service");
+  }
+
   /* Stop and delete a pipeline at the pipeline service */
   PipelineServiceClientResponse deletePipeline(IngestionPipeline ingestionPipeline);
 

openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json

@@ -287,5 +287,5 @@
     }
   },
   "additionalProperties": false,
-  "required": ["id", "name", "appType", "className", "scheduleType", "permission", "runtime", "appSchedule"]
+  "required": ["id", "name", "appType", "className", "scheduleType", "permission", "runtime"]
 }

openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json

@@ -33,6 +33,10 @@
         },
         {
           "$ref": "internal/autoPilotAppConfig.json"
+        },
+        {
+          "type": "object",
+          "additionalProperties": true
         }
       ]
     },
@@ -40,6 +44,10 @@
       "oneOf": [
         {
           "$ref": "private/external/collateAIAppPrivateConfig.json"
+        },
+        {
+          "type": "object",
+          "additionalProperties": true
         }
       ]
     }

openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/helloPipelinesConfiguration.json

@@ -0,0 +1,21 @@
+{
+  "$id": "https://open-metadata.org/schema/entity/applications/configuration/helloPipeliesConfiguration.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Hello Pipelines App Configuration",
+  "description": "This is an example external application.",
+  "properties": {
+    "sleep": {
+      "title": "Sleep time (seconds)",
+      "type": "integer"
+    },
+    "echo": {
+      "title": "Echo message",
+      "type": "string"
+    }
+  },
+  "required": [
+    "sleep",
+    "echo"
+  ],
+  "additionalProperties": false
+}

openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json

@@ -90,7 +90,7 @@
       "type": "string"
     },
     "className": {
-      "description": "Full Qualified ClassName for the the application",
+      "description": "Full Qualified ClassName for the the application. Use can use 'org.openmetadata.service.apps.AbstractNativeApplication' if you don't have one yet.",
       "type": "string"
     },
     "sourcePythonClass": {

openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json

@@ -53,7 +53,7 @@
       "type": "string"
     },
     "className": {
-      "description": "Full Qualified ClassName for the the application",
+      "description": "Full Qualified ClassName for the the application. Use can use 'org.openmetadata.service.apps.AbstractNativeApplication' if you don't have one yet.",
       "type": "string"
     },
     "sourcePythonClass": {
@@ -123,6 +123,11 @@
       "items": {
         "$ref": "../../../events/api/createEventSubscription.json"
       }
+    },
+    "enabled": {
+      "description": "The app will be installable only if this flag is set to true.",
+      "type": "boolean",
+      "default": true
     }
   },
   "additionalProperties": false,

openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/HelloPipelines.md

@@ -0,0 +1,17 @@
+# Hello Pipelines
+
+This schema defines configuration for an example eternal application
+
+$$section
+### sleep $(id="sleep")
+
+Number of seconds to sleep before returning the response
+
+$$
+
+$$section
+### echo $(id="echo")
+
+String to return in the response
+
+$$

openmetadata-ui/src/main/resources/ui/src/generated/configuration/opertionalConfiguration.ts

@@ -37,15 +37,15 @@ export interface SMTPSettings {
     /**
      * Mail of the sender
      */
-    senderMail: string;
+    senderMail?: string;
     /**
      * Smtp Server Endpoint
      */
-    serverEndpoint: string;
+    serverEndpoint?: string;
     /**
      * Smtp Server Port
      */
-    serverPort: number;
+    serverPort?: number;
     /**
      * Support Url
      */

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts

@@ -26,7 +26,7 @@ export interface App {
     /**
      * Application Configuration object.
      */
-    appConfiguration?: any[] | boolean | CollateAIAppConfig | number | null | string;
+    appConfiguration?: any[] | boolean | number | null | CollateAIAppConfig | string;
     /**
      * Application Logo Url.
      */
@@ -34,7 +34,7 @@ export interface App {
     /**
      * In case the app supports scheduling, list of different app schedules
      */
-    appSchedule: any[] | boolean | AppScheduleClass | number | number | null | string;
+    appSchedule?: any[] | boolean | AppScheduleClass | number | number | null | string;
     /**
      * Application Screenshots.
      */
@@ -309,6 +309,7 @@ export interface CollateAIAppConfig {
      * Service Entity Link for which to trigger the application.
      */
     entityLink?: string;
+    [property: string]: any;
 }
 
 /**
@@ -1262,19 +1263,20 @@ export interface PrivateConfig {
      * Collate Server public URL. WAII will use this information to interact with the server.
      * E.g., https://sandbox.getcollate.io
      */
-    collateURL: string;
+    collateURL?: string;
     /**
      * Limits for the CollateAI Application.
      */
-    limits: AppLimitsConfig;
+    limits?: AppLimitsConfig;
     /**
      * WAII API Token
      */
-    token: string;
+    token?: string;
     /**
      * WAII API host URL
      */
-    waiiInstance: string;
+    waiiInstance?: string;
+    [property: string]: any;
 }
 
 /**

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/appRunRecord.ts

@@ -86,7 +86,8 @@ export interface FailureContext {
 }
 
 /**
- * This schema defines Event Publisher Job Error Schema.
+ * This schema defines Event Publisher Job Error Schema. Additional properties exist for
+ * backward compatibility. Don't use it.
  */
 export interface IndexingAppError {
     errorSource?:      ErrorSource;
@@ -98,6 +99,7 @@ export interface IndexingAppError {
     stackTrace?:       string;
     submittedCount?:   number;
     successCount?:     number;
+    [property: string]: any;
 }
 
 export enum ErrorSource {

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/helloPipelinesConfiguration.ts

@@ -0,0 +1,16 @@
+/*
+ *  Copyright 2025 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+export interface HelloPipelinesConfigurationClass {
+    echo:  string;
+    sleep: number;
+}

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts

@@ -26,7 +26,7 @@ export interface AppMarketPlaceDefinition {
     /**
      * Application Configuration object.
      */
-    appConfiguration?: any[] | boolean | CollateAIAppConfig | number | null | string;
+    appConfiguration?: any[] | boolean | number | null | CollateAIAppConfig | string;
     /**
      * Application Logo Url.
      */
@@ -44,7 +44,8 @@ export interface AppMarketPlaceDefinition {
      */
     changeDescription?: ChangeDescription;
     /**
-     * Full Qualified ClassName for the the application
+     * Full Qualified ClassName for the the application. Use can use
+     * 'org.openmetadata.service.apps.AbstractNativeApplication' if you don't have one yet.
      */
     className: string;
     /**
@@ -294,6 +295,7 @@ export interface CollateAIAppConfig {
      * Service Entity Link for which to trigger the application.
      */
     entityLink?: string;
+    [property: string]: any;
 }
 
 /**

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts

@@ -26,7 +26,7 @@ export interface CreateAppMarketPlaceDefinitionReq {
     /**
      * Application Configuration object.
      */
-    appConfiguration?: any[] | boolean | CollateAIAppConfig | number | null | string;
+    appConfiguration?: any[] | boolean | number | null | CollateAIAppConfig | string;
     /**
      * Application Logo Url.
      */
@@ -40,7 +40,8 @@ export interface CreateAppMarketPlaceDefinitionReq {
      */
     appType: AppType;
     /**
-     * Full Qualified ClassName for the the application
+     * Full Qualified ClassName for the the application. Use can use
+     * 'org.openmetadata.service.apps.AbstractNativeApplication' if you don't have one yet.
      */
     className: string;
     /**
@@ -63,6 +64,10 @@ export interface CreateAppMarketPlaceDefinitionReq {
      * Fully qualified name of the domain the Table belongs to.
      */
     domain?: string;
+    /**
+     * The app will be installable only if this flag is set to true.
+     */
+    enabled?: boolean;
     /**
      * Event subscriptions that will be created when the application is installed.
      */
@@ -251,6 +256,7 @@ export interface CollateAIAppConfig {
      * Service Entity Link for which to trigger the application.
      */
     entityLink?: string;
+    [property: string]: any;
 }
 
 /**

openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/HelloPipelines.json

@@ -0,0 +1,17 @@
+{
+  "$id": "HelloPipelines.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Hello Pipelines",
+  "description": "This schema defines configuration for an example eternal application",
+  "properties": {
+    "sleep": {
+      "type": "integer",
+      "description": "Number of seconds to sleep before returning the response"
+    },
+    "echo": {
+      "type": "string",
+      "description": "String to return in the response"
+    }
+  },
+  "additionalProperties": false
+}