diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 1ebcac8d66f..c575f722ba6 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -527,6 +527,8 @@ services: # To integrate GCP AIRFLOW__OPENMETADATA_SECRETS_MANAGER__GCP_PROJECT_ID: ${OM_SM_PROJECT_ID:-""} + # Apps + ENABLE_APP_HelloPipelines: "true" entrypoint: /bin/bash command: diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index eb6e2544140..ff827fe9ab1 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -281,3 +281,5 @@ reportDeprecated = false reportMissingTypeStubs = false reportAny = false reportExplicitAny = false +# @override was only added in python 3.12: https://docs.python.org/3/library/typing.html#typing.override +reportImplicitOverride = false \ No newline at end of file diff --git a/ingestion/src/metadata/applications/example.py b/ingestion/src/metadata/applications/example.py new file mode 100644 index 00000000000..f0732672cad --- /dev/null +++ b/ingestion/src/metadata/applications/example.py @@ -0,0 +1,74 @@ +# Copyright 2025 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Example external application +""" +from time import sleep +from typing import Any + +from metadata.generated.schema.entity.applications.configuration.internal.helloPipelinesConfiguration import ( + HelloPipelinesAppConfiguration, +) +from metadata.generated.schema.metadataIngestion.application import ( + OpenMetadataApplicationConfig, +) +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.logger import app_logger +from metadata.workflow.application import AppRunner, InvalidAppConfiguration + +logger = app_logger() + + +class HelloPipelines(AppRunner): + """ + Example external application that sleeps for a given time and then echoes a message. + You can execute it with `metadata app -c ` + with a YAML file like: + + sourcePythonClass: metadata.applications.example.HelloPipelines + appConfig: + type: HelloPipelines + sleep: 5 + echo: this will be echoed + workflowConfig: + loggerLevel: INFO + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "..." + """ + + def __init__( + self, config: OpenMetadataApplicationConfig, metadata: OpenMetadata[Any, Any] + ): + super().__init__(config, metadata) # pyright: ignore [reportUnknownMemberType] + try: + self.app_config: HelloPipelinesAppConfiguration = ( + HelloPipelinesAppConfiguration.model_validate(self.app_config) + ) + except Exception as e: + raise InvalidAppConfiguration( + f"Hello pipelines received invalid configuration: {e}" + ) + + @property + def name(self) -> str: + return "HelloPipelines" + + def run(self) -> None: + logger.info(f"sleeping for {self.app_config.sleep}") + sleep(self.app_config.sleep) + logger.info("echoing") + logger.info(self.app_config.echo) + + def close(self) -> None: + """Nothing to close""" diff --git a/openmetadata-airflow-apis/README.md b/openmetadata-airflow-apis/README.md index 756cf8e4f95..7260bbd8f5a 100644 --- a/openmetadata-airflow-apis/README.md +++ b/openmetadata-airflow-apis/README.md @@ -5,11 +5,15 @@ OpenMetadata workflow definition and manage DAGS and tasks. ## Development -You can run `make branch=issue-3659-v2 test_up` and specify any branch from OpenMetadata that you'd -need to test the changes in the APIs. This will prepare a separated airflow container. +The file [`development/airflow/airflow.cfg`](./development/airflow/airflow.cfg) contains configuration which runs based on +the airflow server deployed by the quick-start and development compose files. -The command will build the image by downloading the branch changes inside the container. This helps us -test the REST APIs using some ongoing changes on OpenMetadata as well. +You ca run the following command to start the development environment: + +```bash +export AIRFLOW_HOME=$(pwd)/openmetadata-airflow-managed-api/development/airflow +airflow webserver +``` ## Requirements diff --git a/openmetadata-airflow-apis/development/airflow/.gitignore b/openmetadata-airflow-apis/development/airflow/.gitignore new file mode 100644 index 00000000000..f67ac988352 --- /dev/null +++ b/openmetadata-airflow-apis/development/airflow/.gitignore @@ -0,0 +1,5 @@ +logs +dags +dag_generated_configs +airflow-webserver.pid +webserver_config.py diff --git a/openmetadata-airflow-apis/development/airflow/airflow.cfg b/openmetadata-airflow-apis/development/airflow/airflow.cfg new file mode 100644 index 00000000000..14f223bdc3f --- /dev/null +++ b/openmetadata-airflow-apis/development/airflow/airflow.cfg @@ -0,0 +1,40 @@ +[database] +# The SQLAlchemy connection string to the metadata database. +sql_alchemy_conn = mysql+mysqldb://airflow_user:airflow_pass@127.0.0.1:3306/airflow_db +sql_engine_encoding = utf-8 +sql_alchemy_pool_enabled = True +sql_alchemy_pool_size = 5 +sql_alchemy_max_overflow = 10 +sql_alchemy_pool_recycle = 1800 +sql_alchemy_pool_pre_ping = True + +[api] +enable_experimental_api = True +access_control_allow_headers = * +access_control_allow_methods = * +access_control_allow_origins = * +auth_backends = airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session + +[webserver] +web_server_host = 0.0.0.0 +web_server_port = 8080 +expose_config = True +expose_hostname = True +expose_stacktrace = True +workers = 1 +threaded = True + +[core] +dags_are_paused_at_creation = False +load_examples = False +executor = LocalExecutor +parallelism = 1 +max_active_tasks_per_dag = 1 +max_active_runs_per_dag = 1 + +[logging] +logging_level = DEBUG +fab_logging_level = DEBUG + +[openmetadata_airflow_apis] +dag_generated_configs = ${AIRFLOW_HOME}/dag_generated_configs diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/trigger.py b/openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/trigger.py index 8b164c696f9..4d7cfe3eb4e 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/trigger.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/trigger.py @@ -16,7 +16,11 @@ from typing import Callable from flask import Blueprint, Response, request from openmetadata_managed_apis.api.response import ApiResponse -from openmetadata_managed_apis.api.utils import get_request_arg, get_request_dag_id +from openmetadata_managed_apis.api.utils import ( + get_request_arg, + get_request_conf, + get_request_dag_id, +) from openmetadata_managed_apis.operations.trigger import trigger from openmetadata_managed_apis.utils.logger import routes_logger @@ -41,13 +45,14 @@ def get_fn(blueprint: Blueprint) -> Callable: @security.requires_access([(permissions.ACTION_CAN_EDIT, permissions.RESOURCE_DAG)]) def trigger_dag() -> Response: """ - Trigger a dag run + Trigger a dag run with optional configuration """ dag_id = get_request_dag_id() try: run_id = get_request_arg(request, "run_id", raise_missing=False) - response = trigger(dag_id, run_id) + conf = get_request_conf() + response = trigger(dag_id, run_id, conf=conf) return response diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/api/utils.py b/openmetadata-airflow-apis/openmetadata_managed_apis/api/utils.py index d094a80e0c0..c465aa8c5cc 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/api/utils.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/api/utils.py @@ -99,6 +99,16 @@ def get_request_dag_id() -> Optional[str]: return clean_dag_id(raw_dag_id) +def get_request_conf() -> Optional[dict]: + """ + Try to fetch the conf from the JSON request. Return None if no conf is provided. + """ + try: + return request.get_json().get("conf") + except Exception: + return None + + def get_dagbag(): """ Load the dagbag from Airflow settings diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/operations/deploy.py b/openmetadata-airflow-apis/openmetadata_managed_apis/operations/deploy.py index 28cc454caa5..a0a1ad4c9c8 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/operations/deploy.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/operations/deploy.py @@ -97,6 +97,8 @@ class DagDeployer: Store the airflow pipeline config in a JSON file and return the path for the Jinja rendering. """ + # Create directory if it doesn't exist + dag_config_file_path.parent.mkdir(parents=True, exist_ok=True) logger.info(f"Saving file to {dag_config_file_path}") with open(dag_config_file_path, "w") as outfile: diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/operations/trigger.py b/openmetadata-airflow-apis/openmetadata_managed_apis/operations/trigger.py index d6019f2319c..7db76b4c3bb 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/operations/trigger.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/operations/trigger.py @@ -17,17 +17,20 @@ try: from airflow.api.common.trigger_dag import trigger_dag except ImportError: from airflow.api.common.experimental.trigger_dag import trigger_dag + from airflow.utils import timezone from flask import Response from openmetadata_managed_apis.api.response import ApiResponse -def trigger(dag_id: str, run_id: Optional[str]) -> Response: +def trigger( + dag_id: str, run_id: Optional[str], conf: Optional[dict] = None +) -> Response: dag_run = trigger_dag( dag_id=dag_id, run_id=run_id, - conf=None, execution_date=timezone.utcnow(), + conf=conf, ) return ApiResponse.success( {"message": f"Workflow [{dag_id}] has been triggered {dag_run}"} diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py index 6ba8b1fa7dd..b8620fa556b 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py @@ -37,9 +37,7 @@ from metadata.generated.schema.metadataIngestion.applicationPipeline import ( from metadata.workflow.application import ApplicationWorkflow -def application_workflow( - workflow_config: OpenMetadataApplicationConfig, -): +def application_workflow(workflow_config: OpenMetadataApplicationConfig, **context): """ Task that creates and runs the ingestion workflow. @@ -51,9 +49,15 @@ def application_workflow( set_operator_logger(workflow_config) + # set overridden app config config = json.loads( workflow_config.model_dump_json(exclude_defaults=False, mask_secrets=False) ) + params = context.get("params") or {} + config["appConfig"] = { + **(config.get("appConfig") or {}), + **(params.get("appConfigOverride") or {}), + } workflow = ApplicationWorkflow.create(config) execute_workflow(workflow, workflow_config) @@ -100,6 +104,9 @@ def build_application_dag(ingestion_pipeline: IngestionPipeline) -> DAG: ingestion_pipeline=ingestion_pipeline, workflow_config=application_workflow_config, workflow_fn=application_workflow, + params={ + "appConfigOverride": None # Default to None, will be overridden by trigger conf + }, ) return dag diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py index 8f550e8055f..6896c500419 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/common.py @@ -369,9 +369,16 @@ def build_dag( ingestion_pipeline: IngestionPipeline, workflow_config: Union[OpenMetadataWorkflowConfig, OpenMetadataApplicationConfig], workflow_fn: Callable, + params: Optional[dict] = None, ) -> DAG: """ Build a simple metadata workflow DAG + :param task_name: Name of the task + :param ingestion_pipeline: Pipeline configs + :param workflow_config: Workflow configurations + :param workflow_fn: Function to be executed + :param params: Optional parameters to pass to the operator + :return: DAG """ with DAG(**build_dag_configs(ingestion_pipeline)) as dag: @@ -393,6 +400,7 @@ def build_dag( owner=ingestion_pipeline.owners.root[0].name if (ingestion_pipeline.owners and ingestion_pipeline.owners.root) else "openmetadata", + params=params, ) return dag diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/MeteredPipelineServiceClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/MeteredPipelineServiceClient.java index 6b65136f670..0db8d8d78b6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/MeteredPipelineServiceClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/MeteredPipelineServiceClient.java @@ -80,6 +80,15 @@ public class MeteredPipelineServiceClient implements PipelineServiceClientInterf RUN, () -> this.decoratedClient.runPipeline(ingestionPipeline, service)); } + @Override + public PipelineServiceClientResponse runPipeline( + IngestionPipeline ingestionPipeline, + ServiceEntityInterface service, + Map config) { + return this.respondWithMetering( + RUN, () -> this.decoratedClient.runPipeline(ingestionPipeline, service, config)); + } + @Override public PipelineServiceClientResponse deletePipeline(IngestionPipeline ingestionPipeline) { return this.respondWithMetering( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/airflow/AirflowRESTClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/airflow/AirflowRESTClient.java index e88fc0ea25f..9caeb6b1e93 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/airflow/AirflowRESTClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/clients/pipeline/airflow/AirflowRESTClient.java @@ -63,6 +63,8 @@ public class AirflowRESTClient extends PipelineServiceClient { protected final URL serviceURL; private static final List API_ENDPOINT_SEGMENTS = List.of("api", "v1", "openmetadata"); private static final String DAG_ID = "dag_id"; + private static final String CONF = "conf"; + private static final String APP_CONFIG_OVERRIDE = "appConfigOverride"; public AirflowRESTClient(PipelineServiceClientConfiguration config) throws KeyStoreException { @@ -172,12 +174,23 @@ public class AirflowRESTClient extends PipelineServiceClient { @Override public PipelineServiceClientResponse runPipeline( IngestionPipeline ingestionPipeline, ServiceEntityInterface service) { + return runPipeline(ingestionPipeline, service, null); + } + + @Override + public PipelineServiceClientResponse runPipeline( + IngestionPipeline ingestionPipeline, + ServiceEntityInterface service, + Map config) { String pipelineName = ingestionPipeline.getName(); HttpResponse response; try { String triggerUrl = buildURI("trigger").build().toString(); JSONObject requestPayload = new JSONObject(); requestPayload.put(DAG_ID, pipelineName); + if (config != null) { + requestPayload.put(CONF, Map.of(APP_CONFIG_OVERRIDE, config)); + } response = post(triggerUrl, requestPayload.toString()); if (response.statusCode() == 200) { return getResponse(200, response.body()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java index 15c95c61295..8653d704de6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java @@ -5,6 +5,7 @@ import static org.openmetadata.service.jdbi3.EntityRepository.validateOwners; import java.util.List; import java.util.UUID; +import javax.validation.ConstraintViolationException; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; @@ -12,10 +13,12 @@ import org.openmetadata.schema.entity.app.CreateApp; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.BadRequestException; import org.openmetadata.service.jdbi3.AppMarketPlaceRepository; import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.mapper.EntityMapper; import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.JsonUtils; public class AppMapper implements EntityMapper { @Override @@ -65,6 +68,11 @@ public class AppMapper implements EntityMapper { private void validateAndAddBot(App app, String botName) { AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); + try { + JsonUtils.validateJsonSchema(app, App.class); + } catch (ConstraintViolationException e) { + throw BadRequestException.of("Invalid App: " + e.getMessage()); + } if (!CommonUtil.nullOrEmpty(botName)) { app.setBot(Entity.getEntityReferenceByName(BOT, botName, Include.NON_DELETED)); } else { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java index 0facaceb95f..18c1fcdf04f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java @@ -1,5 +1,7 @@ package org.openmetadata.service.resources.apps; +import java.util.Objects; +import javax.validation.ConstraintViolationException; import javax.ws.rs.BadRequestException; import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; import org.openmetadata.schema.entity.app.AppType; @@ -48,16 +50,21 @@ public class AppMarketPlaceMapper } private void validateApplication(AppMarketPlaceDefinition app) { - // Check if the className Exists in classPath - if (app.getAppType().equals(AppType.Internal)) { - // Check class name exists - try { - Class.forName(app.getClassName()); - } catch (ClassNotFoundException e) { - throw new BadRequestException( - "Application Cannot be registered, because the classname cannot be found on the Classpath."); - } - } else { + try { + JsonUtils.validateJsonSchema(app, AppMarketPlaceDefinition.class); + Class.forName( + Objects.requireNonNull( + app.getClassName(), "AppMarketPlaceDefinition.className cannot be null")); + } catch (ClassNotFoundException e) { + throw new BadRequestException( + "Application Cannot be registered, because the Class cannot be found on the Classpath: " + + app.getEventSubscriptions()); + } catch (ConstraintViolationException | NullPointerException e) { + throw new BadRequestException( + "Application Cannot be registered, because the AppMarketPlaceDefinition is not valid: " + + e.getMessage()); + } + if (app.getAppType().equals(AppType.External)) { PipelineServiceClientResponse response = pipelineServiceClient.validateAppRegistration(app); if (response.getCode() != 200) { throw new BadRequestException( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index 1a73adaf9ab..5aa2e4d1627 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -1065,12 +1065,8 @@ public class AppResource extends EntityResource { IngestionPipeline ingestionPipeline = getIngestionPipeline(uriInfo, securityContext, app); ServiceEntityInterface service = Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED); - if (configPayload != null) { - throw new BadRequestException( - "Overriding app config is not supported for external applications."); - } PipelineServiceClientResponse response = - pipelineServiceClient.runPipeline(ingestionPipeline, service); + pipelineServiceClient.runPipeline(ingestionPipeline, service, configPayload); return Response.status(response.getCode()).entity(response).build(); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java index 568749c23d1..2a544e750f8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java @@ -7,6 +7,10 @@ import static org.openmetadata.service.jdbi3.EntityRepository.getEntitiesFromSee import java.io.IOException; import java.util.List; +import java.util.Objects; +import java.util.Optional; +import javax.validation.ConstraintViolationException; +import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; import org.openmetadata.schema.entity.app.CreateAppMarketPlaceDefinitionReq; import org.openmetadata.schema.entity.teams.Role; @@ -20,6 +24,7 @@ import org.openmetadata.service.jdbi3.RoleRepository; import org.openmetadata.service.jdbi3.TeamRepository; import org.openmetadata.service.resources.apps.AppMarketPlaceMapper; +@Slf4j public class AppMarketPlaceUtil { public static void createAppMarketPlaceDefinitions( AppMarketPlaceRepository appMarketRepository, AppMarketPlaceMapper mapper) @@ -46,15 +51,35 @@ public class AppMarketPlaceUtil { teamRepository.initOrganization(); } - List createAppMarketPlaceDefinitionReqs = - getEntitiesFromSeedData( + getEntitiesFromSeedData( APPLICATION, String.format(".*json/data/%s/.*\\.json$", Entity.APP_MARKET_PLACE_DEF), - CreateAppMarketPlaceDefinitionReq.class); - for (CreateAppMarketPlaceDefinitionReq definitionReq : createAppMarketPlaceDefinitionReqs) { - AppMarketPlaceDefinition definition = mapper.createToEntity(definitionReq, ADMIN_USER_NAME); - appMarketRepository.setFullyQualifiedName(definition); - appMarketRepository.createOrUpdate(null, definition, ADMIN_USER_NAME); - } + CreateAppMarketPlaceDefinitionReq.class) + .stream() + .filter( + req -> + Optional.ofNullable(System.getenv("ENABLE_APP_" + req.getName())) + .map(val -> Objects.equals(val, "true")) + .orElse(req.getEnabled())) + .filter( + req -> { + try { + JsonUtils.validateJsonSchema(req, CreateAppMarketPlaceDefinitionReq.class); + return true; + } catch (ConstraintViolationException e) { + LOG.error( + "Error validating {}: {}", + CreateAppMarketPlaceDefinitionReq.class.getSimpleName(), + req.getName(), + e); + return false; + } + }) + .forEach( + req -> { + AppMarketPlaceDefinition definition = mapper.createToEntity(req, ADMIN_USER_NAME); + appMarketRepository.setFullyQualifiedName(definition); + appMarketRepository.createOrUpdate(null, definition, ADMIN_USER_NAME); + }); } } diff --git a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/HelloPipelines.json b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/HelloPipelines.json new file mode 100644 index 00000000000..856954e9442 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/HelloPipelines.json @@ -0,0 +1,25 @@ +{ + "name": "HelloPipelines", + "displayName": "Hello Pipelines", + "description": "Example external application to demonstrate the use of pipelines.", + "features": "Test external application functionality", + "appType": "external", + "appScreenshots": ["DataInsightsPic1.png"], + "developer": "Collate Inc.", + "developerUrl": "https://www.getcollate.io", + "privacyPolicyUrl": "https://www.getcollate.io", + "supportEmail": "support@getcollate.io", + "scheduleType": "ScheduledOrManual", + "permission": "All", + "className": "org.openmetadata.service.apps.AbstractNativeApplication", + "sourcePythonClass": "metadata.applications.example.HelloPipelines", + "runtime": { + "enabled": "true" + }, + "appConfiguration": { + "type": "HelloPipelines", + "sleep": 10, + "echo": "hello pipelines" + }, + "enabled": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClientInterface.java b/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClientInterface.java index 3de12be69c7..421664c2761 100644 --- a/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClientInterface.java +++ b/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClientInterface.java @@ -105,6 +105,16 @@ public interface PipelineServiceClientInterface { PipelineServiceClientResponse runPipeline( IngestionPipeline ingestionPipeline, ServiceEntityInterface service); + /* Deploy run the pipeline at the pipeline service with ad-hoc custom configuration. + * This might not be supported by some pipeline service clients.*/ + default PipelineServiceClientResponse runPipeline( + IngestionPipeline ingestionPipeline, + ServiceEntityInterface service, + Map config) { + throw new UnsupportedOperationException( + "This operation is not supported by this pipeline service"); + } + /* Stop and delete a pipeline at the pipeline service */ PipelineServiceClientResponse deletePipeline(IngestionPipeline ingestionPipeline); diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json index 78a0dab0607..8fcc736786c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json @@ -287,5 +287,5 @@ } }, "additionalProperties": false, - "required": ["id", "name", "appType", "className", "scheduleType", "permission", "runtime", "appSchedule"] + "required": ["id", "name", "appType", "className", "scheduleType", "permission", "runtime"] } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json index 318346912fa..52abd66d081 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json @@ -33,6 +33,10 @@ }, { "$ref": "internal/autoPilotAppConfig.json" + }, + { + "type": "object", + "additionalProperties": true } ] }, @@ -40,6 +44,10 @@ "oneOf": [ { "$ref": "private/external/collateAIAppPrivateConfig.json" + }, + { + "type": "object", + "additionalProperties": true } ] } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/helloPipelinesConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/helloPipelinesConfiguration.json new file mode 100644 index 00000000000..9e75f5a87a4 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/helloPipelinesConfiguration.json @@ -0,0 +1,21 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/configuration/helloPipeliesConfiguration.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Hello Pipelines App Configuration", + "description": "This is an example external application.", + "properties": { + "sleep": { + "title": "Sleep time (seconds)", + "type": "integer" + }, + "echo": { + "title": "Echo message", + "type": "string" + } + }, + "required": [ + "sleep", + "echo" + ], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json index 95bfde1dad8..f2244eeecd0 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json @@ -90,7 +90,7 @@ "type": "string" }, "className": { - "description": "Full Qualified ClassName for the the application", + "description": "Full Qualified ClassName for the the application. Use can use 'org.openmetadata.service.apps.AbstractNativeApplication' if you don't have one yet.", "type": "string" }, "sourcePythonClass": { diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json index 9909df71a69..c276b82b77a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json @@ -53,7 +53,7 @@ "type": "string" }, "className": { - "description": "Full Qualified ClassName for the the application", + "description": "Full Qualified ClassName for the the application. Use can use 'org.openmetadata.service.apps.AbstractNativeApplication' if you don't have one yet.", "type": "string" }, "sourcePythonClass": { @@ -123,6 +123,11 @@ "items": { "$ref": "../../../events/api/createEventSubscription.json" } + }, + "enabled": { + "description": "The app will be installable only if this flag is set to true.", + "type": "boolean", + "default": true } }, "additionalProperties": false, diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/HelloPipelines.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/HelloPipelines.md new file mode 100644 index 00000000000..706dd313a57 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/HelloPipelines.md @@ -0,0 +1,17 @@ +# Hello Pipelines + +This schema defines configuration for an example eternal application + +$$section +### sleep $(id="sleep") + +Number of seconds to sleep before returning the response + +$$ + +$$section +### echo $(id="echo") + +String to return in the response + +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/opertionalConfiguration.ts b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/opertionalConfiguration.ts index 990a042e0c6..38179085571 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/opertionalConfiguration.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/opertionalConfiguration.ts @@ -37,15 +37,15 @@ export interface SMTPSettings { /** * Mail of the sender */ - senderMail: string; + senderMail?: string; /** * Smtp Server Endpoint */ - serverEndpoint: string; + serverEndpoint?: string; /** * Smtp Server Port */ - serverPort: number; + serverPort?: number; /** * Support Url */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts index 21ee4953c37..e7ad973eb40 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts @@ -26,7 +26,7 @@ export interface App { /** * Application Configuration object. */ - appConfiguration?: any[] | boolean | CollateAIAppConfig | number | null | string; + appConfiguration?: any[] | boolean | number | null | CollateAIAppConfig | string; /** * Application Logo Url. */ @@ -34,7 +34,7 @@ export interface App { /** * In case the app supports scheduling, list of different app schedules */ - appSchedule: any[] | boolean | AppScheduleClass | number | number | null | string; + appSchedule?: any[] | boolean | AppScheduleClass | number | number | null | string; /** * Application Screenshots. */ @@ -309,6 +309,7 @@ export interface CollateAIAppConfig { * Service Entity Link for which to trigger the application. */ entityLink?: string; + [property: string]: any; } /** @@ -1262,19 +1263,20 @@ export interface PrivateConfig { * Collate Server public URL. WAII will use this information to interact with the server. * E.g., https://sandbox.getcollate.io */ - collateURL: string; + collateURL?: string; /** * Limits for the CollateAI Application. */ - limits: AppLimitsConfig; + limits?: AppLimitsConfig; /** * WAII API Token */ - token: string; + token?: string; /** * WAII API host URL */ - waiiInstance: string; + waiiInstance?: string; + [property: string]: any; } /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/appRunRecord.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/appRunRecord.ts index ed5b2a0c595..3adebcdf977 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/appRunRecord.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/appRunRecord.ts @@ -86,7 +86,8 @@ export interface FailureContext { } /** - * This schema defines Event Publisher Job Error Schema. + * This schema defines Event Publisher Job Error Schema. Additional properties exist for + * backward compatibility. Don't use it. */ export interface IndexingAppError { errorSource?: ErrorSource; @@ -98,6 +99,7 @@ export interface IndexingAppError { stackTrace?: string; submittedCount?: number; successCount?: number; + [property: string]: any; } export enum ErrorSource { diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/helloPipelinesConfiguration.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/helloPipelinesConfiguration.ts new file mode 100644 index 00000000000..a94cff032b2 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/helloPipelinesConfiguration.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export interface HelloPipelinesConfigurationClass { + echo: string; + sleep: number; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts index 5732667f792..1d19168d6c4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts @@ -26,7 +26,7 @@ export interface AppMarketPlaceDefinition { /** * Application Configuration object. */ - appConfiguration?: any[] | boolean | CollateAIAppConfig | number | null | string; + appConfiguration?: any[] | boolean | number | null | CollateAIAppConfig | string; /** * Application Logo Url. */ @@ -44,7 +44,8 @@ export interface AppMarketPlaceDefinition { */ changeDescription?: ChangeDescription; /** - * Full Qualified ClassName for the the application + * Full Qualified ClassName for the the application. Use can use + * 'org.openmetadata.service.apps.AbstractNativeApplication' if you don't have one yet. */ className: string; /** @@ -294,6 +295,7 @@ export interface CollateAIAppConfig { * Service Entity Link for which to trigger the application. */ entityLink?: string; + [property: string]: any; } /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts index 41dcf70c0c9..371f0d69ac9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts @@ -26,7 +26,7 @@ export interface CreateAppMarketPlaceDefinitionReq { /** * Application Configuration object. */ - appConfiguration?: any[] | boolean | CollateAIAppConfig | number | null | string; + appConfiguration?: any[] | boolean | number | null | CollateAIAppConfig | string; /** * Application Logo Url. */ @@ -40,7 +40,8 @@ export interface CreateAppMarketPlaceDefinitionReq { */ appType: AppType; /** - * Full Qualified ClassName for the the application + * Full Qualified ClassName for the the application. Use can use + * 'org.openmetadata.service.apps.AbstractNativeApplication' if you don't have one yet. */ className: string; /** @@ -63,6 +64,10 @@ export interface CreateAppMarketPlaceDefinitionReq { * Fully qualified name of the domain the Table belongs to. */ domain?: string; + /** + * The app will be installable only if this flag is set to true. + */ + enabled?: boolean; /** * Event subscriptions that will be created when the application is installed. */ @@ -251,6 +256,7 @@ export interface CollateAIAppConfig { * Service Entity Link for which to trigger the application. */ entityLink?: string; + [property: string]: any; } /** diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/HelloPipelines.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/HelloPipelines.json new file mode 100644 index 00000000000..9a967b9eb9a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/HelloPipelines.json @@ -0,0 +1,17 @@ +{ + "$id": "HelloPipelines.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Hello Pipelines", + "description": "This schema defines configuration for an example eternal application", + "properties": { + "sleep": { + "type": "integer", + "description": "Number of seconds to sleep before returning the response" + }, + "echo": { + "type": "string", + "description": "String to return in the response" + } + }, + "additionalProperties": false +}