ISSUE #2031 - Add App Runner to Application Entity (#23259)

* feat: added exporter app config

* refactor: added entityprofile resource & added backward compatibility to existing API

* feat: added tests to get_profile_data_by_type

* feat: remove non supported event types

* chore: added migrations to 1.9.7

* chore: added application creation readme

* feat: implemented backend support for agent runner for external applications

* chore: move migrations to 1.9.8

* Update generated TypeScript types

* fix: added setIngestionRunner method to serviceEntityInterface interface

* Update generated TypeScript types

* fix: remove service instanceof MetadataService in deploy endpoint

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
(cherry picked from commit 568baab4267186490f88237f623cf886ce75b1b8)

openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java

@@ -9,6 +9,7 @@ import javax.validation.ConstraintViolationException;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.app.App;
 import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition;
+import org.openmetadata.schema.entity.app.AppType;
 import org.openmetadata.schema.entity.app.CreateApp;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
@@ -30,6 +31,9 @@ public class AppMapper implements EntityMapper<App, CreateApp> {
             null,
             createAppRequest.getName(),
             new EntityUtil.Fields(appMarketPlaceRepository.getAllowedFields()));
+    Boolean supportsIngestionRunner =
+        !marketPlaceDefinition.getAppType().equals(AppType.Internal)
+            && marketPlaceDefinition.getSupportsIngestionRunner();
     List<EntityReference> owners = validateOwners(createAppRequest.getOwners());
     App app =
         new App()
@@ -59,7 +63,12 @@ public class AppMapper implements EntityMapper<App, CreateApp> {
             .withAllowConfiguration(marketPlaceDefinition.getAllowConfiguration())
             .withSystem(marketPlaceDefinition.getSystem())
             .withSupportsInterrupt(marketPlaceDefinition.getSupportsInterrupt())
-            .withFullyQualifiedName(marketPlaceDefinition.getFullyQualifiedName());
+            .withFullyQualifiedName(marketPlaceDefinition.getFullyQualifiedName())
+            .withSupportsIngestionRunner(supportsIngestionRunner)
+            .withIngestionRunner(
+                supportsIngestionRunner.equals(true)
+                    ? createAppRequest.getIngestionRunner()
+                    : null);
 
     // validate Bot if provided
     validateAndAddBot(app, createAppRequest.getBot());

openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java

@@ -44,7 +44,8 @@ public class AppMarketPlaceMapper
             .withAllowConfiguration(create.getAllowConfiguration())
             .withSystem(create.getSystem())
             .withSupportsInterrupt(create.getSupportsInterrupt())
-            .withEventSubscriptions(create.getEventSubscriptions());
+            .withEventSubscriptions(create.getEventSubscriptions())
+            .withSupportsIngestionRunner(create.getSupportsIngestionRunner());
 
     // Validate App
     validateApplication(app);

openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java

@@ -1080,6 +1080,11 @@ public class AppResource extends EntityResource<App, AppRepository> {
         IngestionPipeline ingestionPipeline = getIngestionPipeline(uriInfo, securityContext, app);
         ServiceEntityInterface service =
             Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED);
+
+        if (app.getSupportsIngestionRunner()) {
+          service.setIngestionRunner(app.getIngestionRunner());
+        }
+
         PipelineServiceClientResponse response =
             pipelineServiceClient.runPipeline(ingestionPipeline, service, configPayload);
         return Response.status(response.getCode()).entity(response).build();
@@ -1165,6 +1170,11 @@ public class AppResource extends EntityResource<App, AppRepository> {
         IngestionPipeline ingestionPipeline = getIngestionPipeline(uriInfo, securityContext, app);
         ServiceEntityInterface service =
             Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED);
+
+        if (app.getSupportsIngestionRunner()) {
+          service.setIngestionRunner(app.getIngestionRunner());
+        }
+
         PipelineServiceClientResponse status =
             pipelineServiceClient.deployPipeline(ingestionPipeline, service);
         if (status.getCode() == 200) {

openmetadata-spec/src/main/java/org/openmetadata/schema/ServiceEntityInterface.java

@@ -29,6 +29,8 @@ public interface ServiceEntityInterface extends EntityInterface {
 
   void setPipelines(List<EntityReference> pipelines);
 
+  void setIngestionRunner(EntityReference ingestionRunner);
+
   void setTestConnectionResult(TestConnectionResult testConnectionResult);
 
   EnumInterface getServiceType();

openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json

@@ -304,6 +304,15 @@
     "eventSubscriptions": {
       "description": "Event Subscriptions for the Application.",
       "$ref": "../../type/entityReferenceList.json"
+    },
+    "supportsIngestionRunner": {
+      "description": "If the app support execution through the external runner.",
+      "type": "boolean",
+      "default": false
+    },
+    "ingestionRunner" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline. It will be defined at runtime based on the Ingestion Agent of the service.",
+      "$ref": "../../type/entityReference.json"
     }
   },
   "additionalProperties": false,

openmetadata-spec/src/main/resources/json/schema/entity/applications/createAppRequest.json

@@ -46,6 +46,10 @@
       "description": "If the app run can be interrupted as part of the execution.",
       "type": "boolean",
       "default": false
+    },
+    "ingestionRunner" : {
+      "description": "The ingestion agent responsible for executing the ingestion pipeline. It will be defined at runtime based on the Ingestion Agent of the service.",
+      "$ref": "../../type/entityReference.json"
     }
   },
   "additionalProperties": false

openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json

@@ -165,6 +165,11 @@
       "items": {
         "$ref": "../../../events/api/createEventSubscription.json"
       }
+    },
+    "supportsIngestionRunner": {
+      "description": "If the app support execution through the external runner.",
+      "type": "boolean",
+      "default": false
     }
   },
   "additionalProperties": false,

openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json

@@ -131,6 +131,11 @@
       "description": "The app will be installable only if this flag is set to true.",
       "type": "boolean",
       "default": true
+    },
+    "supportsIngestionRunner": {
+      "description": "If the app support execution through the external runner.",
+      "type": "boolean",
+      "default": false
     }
   },
   "additionalProperties": false,

openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json

@@ -175,6 +175,10 @@
       "description": "Indicates if the test suite is inherited from a parent entity.",
       "type": "boolean",
       "default": false
+    },
+    "ingestionRunner": {
+      "description": "Link to the ingestion pipeline that ingested this entity.",
+      "$ref": "../type/entityReference.json"
     }
   },
   "required": [

openmetadata-ui/src/main/resources/ui/src/generated/api/data/createEntityProfile.ts

@@ -0,0 +1,251 @@
+/*
+ *  Copyright 2025 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+/**
+ * Request schema for creating an entity profile for a table.
+ */
+export interface CreateEntityProfile {
+    /**
+     * Type of entity for which the profile is created. For example, 'table'.
+     */
+    entityType:  string;
+    profileData: Profile;
+    /**
+     * type of profile
+     */
+    profileType?: ProfileTypeEnum;
+    /**
+     * Timestamp when the profile is created.
+     */
+    timestamp?: number;
+}
+
+/**
+ * This schema defines the type to capture the table's data profile.
+ *
+ * This schema defines the type to capture the table's column profile.
+ *
+ * This schema defines the System Profile object holding profile data from system tables.
+ */
+export interface Profile {
+    /**
+     * No.of columns in the table.
+     */
+    columnCount?: number;
+    /**
+     * Table creation time.
+     */
+    createDateTime?: Date;
+    /**
+     * Custom Metrics profile list bound to a column.
+     */
+    customMetrics?: CustomMetricProfile[];
+    /**
+     * Percentage of data or no. of rows we want to execute the profiler and tests on
+     */
+    profileSample?:     number;
+    profileSampleType?: ProfileSampleType;
+    /**
+     * No.of rows in the table. This is always executed on the whole table.
+     */
+    rowCount?:           number;
+    samplingMethodType?: SamplingMethodType;
+    /**
+     * Table size in GB
+     */
+    sizeInByte?: number;
+    /**
+     * Timestamp on which profile is taken.
+     */
+    timestamp?: number;
+    /**
+     * Number of values that contain distinct values.
+     */
+    distinctCount?: number;
+    /**
+     * Proportion of distinct values in a column.
+     */
+    distinctProportion?: number;
+    /**
+     * No.of Rows that contain duplicates in a column.
+     */
+    duplicateCount?: number;
+    /**
+     * First quartile of a column.
+     */
+    firstQuartile?: number;
+    /**
+     * Histogram of a column.
+     */
+    histogram?: any[] | boolean | HistogramClass | number | number | null | string;
+    /**
+     * Inter quartile range of a column.
+     */
+    interQuartileRange?: number;
+    /**
+     * Maximum value in a column.
+     */
+    max?: number | string;
+    /**
+     * Maximum string length in a column.
+     */
+    maxLength?: number;
+    /**
+     * Avg value in a column.
+     */
+    mean?: number;
+    /**
+     * Median of a column.
+     */
+    median?: number;
+    /**
+     * Minimum value in a column.
+     */
+    min?: number | string;
+    /**
+     * Minimum string length in a column.
+     */
+    minLength?: number;
+    /**
+     * Missing count is calculated by subtracting valuesCount - validCount.
+     */
+    missingCount?: number;
+    /**
+     * Missing Percentage is calculated by taking percentage of validCount/valuesCount.
+     */
+    missingPercentage?: number;
+    /**
+     * Column Name.
+     */
+    name?: string;
+    /**
+     * Non parametric skew of a column.
+     */
+    nonParametricSkew?: number;
+    /**
+     * No.of null values in a column.
+     */
+    nullCount?: number;
+    /**
+     * No.of null value proportion in columns.
+     */
+    nullProportion?: number;
+    /**
+     * Standard deviation of a column.
+     */
+    stddev?: number;
+    /**
+     * Median value in a column.
+     */
+    sum?: number;
+    /**
+     * First quartile of a column.
+     */
+    thirdQuartile?: number;
+    /**
+     * No. of unique values in the column.
+     */
+    uniqueCount?: number;
+    /**
+     * Proportion of number of unique values in a column.
+     */
+    uniqueProportion?: number;
+    /**
+     * Total count of valid values in this column.
+     */
+    validCount?: number;
+    /**
+     * Total count of the values in this column.
+     */
+    valuesCount?: number;
+    /**
+     * Percentage of values in this column with respect to row count.
+     */
+    valuesPercentage?: number;
+    /**
+     * Variance of a column.
+     */
+    variance?: number;
+    /**
+     * Operation performed.
+     */
+    operation?: DMLOperationType;
+    /**
+     * Number of rows affected.
+     */
+    rowsAffected?: number;
+    [property: string]: any;
+}
+
+/**
+ * Profiling results of a Custom Metric.
+ */
+export interface CustomMetricProfile {
+    /**
+     * Custom metric name.
+     */
+    name?: string;
+    /**
+     * Profiling results for the metric.
+     */
+    value?: number;
+}
+
+export interface HistogramClass {
+    /**
+     * Boundaries of Histogram.
+     */
+    boundaries?: any[];
+    /**
+     * Frequencies of Histogram.
+     */
+    frequencies?: any[];
+}
+
+/**
+ * Operation performed.
+ *
+ * This schema defines the type of DML operation.
+ */
+export enum DMLOperationType {
+    Delete = "DELETE",
+    Insert = "INSERT",
+    Update = "UPDATE",
+    Write = "WRITE",
+}
+
+/**
+ * Type of Profile Sample (percentage or rows)
+ */
+export enum ProfileSampleType {
+    Percentage = "PERCENTAGE",
+    Rows = "ROWS",
+}
+
+/**
+ * Type of Sampling Method (BERNOULLI or SYSTEM)
+ */
+export enum SamplingMethodType {
+    Bernoulli = "BERNOULLI",
+    System = "SYSTEM",
+}
+
+/**
+ * type of profile
+ *
+ * profile type
+ */
+export enum ProfileTypeEnum {
+    Column = "column",
+    System = "system",
+    Table = "table",
+}

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts

@@ -104,6 +104,11 @@ export interface App {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline. It will be defined
+     * at runtime based on the Ingestion Agent of the service.
+     */
+    ingestionRunner?: EntityReference;
     /**
      * Name of the Application.
      */
@@ -153,6 +158,10 @@ export interface App {
      * Support Email for the application
      */
     supportEmail?: string;
+    /**
+     * If the app support execution through the external runner.
+     */
+    supportsIngestionRunner?: boolean;
     /**
      * If the app run can be interrupted as part of the execution.
      */
@@ -547,6 +556,9 @@ export interface Action {
  * the relationship of a table `belongs to a` database.
  *
  * Bot User Associated with this application.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline. It will be defined
+ * at runtime based on the Ingestion Agent of the service.
  */
 export interface EntityReference {
     /**
@@ -646,6 +658,9 @@ export enum MetadataAttribute {
  * the relationship of a table `belongs to a` database.
  *
  * Bot User Associated with this application.
+ *
+ * The ingestion agent responsible for executing the ingestion pipeline. It will be defined
+ * at runtime based on the Ingestion Agent of the service.
  */
 export interface TagLabel {
     /**

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/metadataExporterAppConfig.ts

@@ -0,0 +1,186 @@
+/*
+ *  Copyright 2025 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+/**
+ * Configuration for the Metadata Exporter Application.
+ */
+export interface MetadataExporterAppConfig {
+    /**
+     * Enable backfill for the exporter to process historical data. This will only work on the
+     * very first run of the exporter.
+     */
+    backfill?: boolean;
+    /**
+     * Connection details for the Metadata Exporter Application.
+     */
+    connectionConfig: SnowflakeConnection;
+    /**
+     * List of event types to export.
+     */
+    eventTypes?: EventType[];
+    /**
+     * Range of data to export. Options are 'ALL' for all data, 'LATEST' for the latest data, or
+     * a specific date range.
+     */
+    exportRange: ExportRangeConfiguration;
+    /**
+     * Configuration for the table to export the data to.
+     */
+    tableConfiguration: ConfigurationForTheTableToExportTheDataTo;
+    /**
+     * Application Type
+     */
+    type?: MetadataExporterAppType;
+}
+
+/**
+ * Connection details for the Metadata Exporter Application.
+ *
+ * Snowflake Connection Config
+ */
+export interface SnowflakeConnection {
+    /**
+     * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the
+     * account is xyz1234.us-east-1.gcp
+     */
+    account: string;
+    /**
+     * Optional configuration for ingestion to keep the client session active in case the
+     * ingestion process runs for longer durations.
+     */
+    clientSessionKeepAlive?: boolean;
+    connectionArguments?:    { [key: string]: any };
+    connectionOptions?:      { [key: string]: string };
+    /**
+     * Database of the data source. This is optional parameter, if you would like to restrict
+     * the metadata reading to a single database. When left blank, OpenMetadata Ingestion
+     * attempts to scan all the databases.
+     */
+    database?: string;
+    /**
+     * Password to connect to Snowflake.
+     */
+    password?: string;
+    /**
+     * Connection to Snowflake instance via Private Key
+     */
+    privateKey?: string;
+    /**
+     * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user
+     * should have enough privileges to alter the session.
+     */
+    queryTag?: string;
+    /**
+     * Snowflake Role.
+     */
+    role?: string;
+    /**
+     * SQLAlchemy driver scheme options.
+     */
+    scheme?: SnowflakeScheme;
+    /**
+     * Snowflake Passphrase Key used with Private Key
+     */
+    snowflakePrivatekeyPassphrase?: string;
+    /**
+     * Service Type
+     */
+    type?: SnowflakeType;
+    /**
+     * Username to connect to Snowflake. This user should have privileges to read all the
+     * metadata in Snowflake.
+     */
+    username: string;
+    /**
+     * Snowflake warehouse.
+     */
+    warehouse: string;
+}
+
+/**
+ * SQLAlchemy driver scheme options.
+ */
+export enum SnowflakeScheme {
+    Snowflake = "snowflake",
+}
+
+/**
+ * Service Type
+ *
+ * Service type.
+ */
+export enum SnowflakeType {
+    Snowflake = "Snowflake",
+}
+
+export enum EventType {
+    Profile = "PROFILE",
+    TestCaseResults = "TEST_CASE_RESULTS",
+}
+
+/**
+ * Range of data to export. Options are 'ALL' for all data, 'LATEST' for the latest data, or
+ * a specific date range.
+ *
+ * Configuration for the export range of data.
+ */
+export interface ExportRangeConfiguration {
+    /**
+     * Interval for the export range, applicable when rangeType is 'DATE_RANGE'.
+     */
+    interval?: number;
+    /**
+     * Type of range for data export.
+     */
+    rangeType: RangeType;
+    /**
+     * Unit of time for the export range.
+     */
+    unit?: Unit;
+    [property: string]: any;
+}
+
+/**
+ * Type of range for data export.
+ */
+export enum RangeType {
+    All = "ALL",
+    DateRange = "DATE_RANGE",
+    Latest = "LATEST",
+}
+
+/**
+ * Unit of time for the export range.
+ */
+export enum Unit {
+    Days = "DAYS",
+    Hours = "HOURS",
+}
+
+/**
+ * Configuration for the table to export the data to.
+ */
+export interface ConfigurationForTheTableToExportTheDataTo {
+    databaseName: string;
+    schemaName:   string;
+    tableName:    string;
+    [property: string]: any;
+}
+
+/**
+ * Application Type
+ *
+ * Application type.
+ */
+export enum MetadataExporterAppType {
+    MetadataExporter = "MetadataExporter",
+}

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/metadataExporterConnectors/bigQueryConnection.ts

@@ -0,0 +1,188 @@
+/*
+ *  Copyright 2025 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+/**
+ * Google BigQuery Connection Config
+ */
+export interface BigQueryConnection {
+    /**
+     * Billing Project ID
+     */
+    billingProjectId?:    string;
+    connectionArguments?: { [key: string]: any };
+    connectionOptions?:   { [key: string]: string };
+    /**
+     * GCP Credentials
+     */
+    credentials: GCPCredentials;
+    /**
+     * BigQuery APIs URL.
+     */
+    hostPort?: string;
+    /**
+     * SQLAlchemy driver scheme options.
+     */
+    scheme?: BigqueryScheme;
+    /**
+     * Taxonomy location used to fetch policy tags
+     */
+    taxonomyLocation?: string;
+    /**
+     * Project IDs used to fetch policy tags
+     */
+    taxonomyProjectID?: string[];
+    /**
+     * Service Type
+     */
+    type?: BigqueryType;
+    /**
+     * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can
+     * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia
+     * multi-regions are not yet in GA.
+     */
+    usageLocation?: string;
+}
+
+/**
+ * GCP Credentials
+ *
+ * GCP credentials configs.
+ */
+export interface GCPCredentials {
+    /**
+     * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP
+     * Credentials Path
+     */
+    gcpConfig: GCPCredentialsConfiguration;
+    /**
+     * we enable the authenticated service account to impersonate another service account
+     */
+    gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues;
+}
+
+/**
+ * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP
+ * Credentials Path
+ *
+ * Pass the raw credential values provided by GCP
+ *
+ * Pass the path of file containing the GCP credentials info
+ *
+ * Use the application default credentials
+ */
+export interface GCPCredentialsConfiguration {
+    /**
+     * Google Cloud auth provider certificate.
+     */
+    authProviderX509CertUrl?: string;
+    /**
+     * Google Cloud auth uri.
+     */
+    authUri?: string;
+    /**
+     * Google Cloud email.
+     */
+    clientEmail?: string;
+    /**
+     * Google Cloud Client ID.
+     */
+    clientId?: string;
+    /**
+     * Google Cloud client certificate uri.
+     */
+    clientX509CertUrl?: string;
+    /**
+     * Google Cloud private key.
+     */
+    privateKey?: string;
+    /**
+     * Google Cloud private key id.
+     */
+    privateKeyId?: string;
+    /**
+     * Project ID
+     *
+     * GCP Project ID to parse metadata from
+     */
+    projectId?: string[] | string;
+    /**
+     * Google Cloud token uri.
+     */
+    tokenUri?: string;
+    /**
+     * Google Cloud Platform account type.
+     *
+     * Google Cloud Platform ADC ( Application Default Credentials )
+     */
+    type?: string;
+    /**
+     * Path of the file containing the GCP credentials info
+     */
+    path?: string;
+    /**
+     * Google Security Token Service audience which contains the resource name for the workload
+     * identity pool and the provider identifier in that pool.
+     */
+    audience?: string;
+    /**
+     * This object defines the mechanism used to retrieve the external credential from the local
+     * environment so that it can be exchanged for a GCP access token via the STS endpoint
+     */
+    credentialSource?: { [key: string]: string };
+    /**
+     * Google Cloud Platform account type.
+     */
+    externalType?: string;
+    /**
+     * Google Security Token Service subject token type based on the OAuth 2.0 token exchange
+     * spec.
+     */
+    subjectTokenType?: string;
+    /**
+     * Google Security Token Service token exchange endpoint.
+     */
+    tokenURL?: string;
+    [property: string]: any;
+}
+
+/**
+ * we enable the authenticated service account to impersonate another service account
+ *
+ * Pass the values to impersonate a service account of Google Cloud
+ */
+export interface GCPImpersonateServiceAccountValues {
+    /**
+     * The impersonated service account email
+     */
+    impersonateServiceAccount?: string;
+    /**
+     * Number of seconds the delegated credential should be valid
+     */
+    lifetime?: number;
+    [property: string]: any;
+}
+
+/**
+ * SQLAlchemy driver scheme options.
+ */
+export enum BigqueryScheme {
+    Bigquery = "bigquery",
+}
+
+/**
+ * Service Type
+ *
+ * Service type.
+ */
+export enum BigqueryType {
+    BigQuery = "BigQuery",
+}

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/metadataExporterConnectors/databricksConnection.ts

@@ -0,0 +1,73 @@
+/*
+ *  Copyright 2025 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+/**
+ * Databricks Connection Config
+ */
+export interface DatabricksConnection {
+    /**
+     * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you
+     * would like to restrict the metadata reading to a single catalog. When left blank,
+     * OpenMetadata Ingestion attempts to scan all the catalog.
+     */
+    catalog?:             string;
+    connectionArguments?: { [key: string]: any };
+    connectionOptions?:   { [key: string]: string };
+    /**
+     * The maximum amount of time (in seconds) to wait for a successful connection to the data
+     * source. If the connection attempt takes longer than this timeout period, an error will be
+     * returned.
+     */
+    connectionTimeout?: number;
+    /**
+     * Database Schema of the data source. This is optional parameter, if you would like to
+     * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion
+     * attempts to scan all the schemas.
+     */
+    databaseSchema?: string;
+    /**
+     * Host and port of the Databricks service.
+     */
+    hostPort: string;
+    /**
+     * Databricks compute resources URL.
+     */
+    httpPath: string;
+    /**
+     * SQLAlchemy driver scheme options.
+     */
+    scheme?: DatabricksScheme;
+    /**
+     * Generated Token to connect to Databricks.
+     */
+    token: string;
+    /**
+     * Service Type
+     */
+    type?: DatabricksType;
+}
+
+/**
+ * SQLAlchemy driver scheme options.
+ */
+export enum DatabricksScheme {
+    DatabricksConnector = "databricks+connector",
+}
+
+/**
+ * Service Type
+ *
+ * Service type.
+ */
+export enum DatabricksType {
+    Databricks = "Databricks",
+}

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/metadataExporterConnectors/redshiftConnection.ts

@@ -0,0 +1,95 @@
+/*
+ *  Copyright 2025 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+/**
+ * Redshift  Connection Config
+ */
+export interface RedshiftConnection {
+    connectionArguments?: { [key: string]: any };
+    connectionOptions?:   { [key: string]: string };
+    /**
+     * Initial Redshift database to connect to. If you want to ingest all databases, set
+     * ingestAllDatabases to true.
+     */
+    database: string;
+    /**
+     * Host and port of the Redshift service.
+     */
+    hostPort: string;
+    /**
+     * Password to connect to Redshift.
+     */
+    password?: string;
+    /**
+     * SQLAlchemy driver scheme options.
+     */
+    scheme?:    RedshiftScheme;
+    sslConfig?: Config;
+    sslMode?:   SSLMode;
+    /**
+     * Service Type
+     */
+    type?: RedshiftType;
+    /**
+     * Username to connect to Redshift. This user should have privileges to read all the
+     * metadata in Redshift.
+     */
+    username: string;
+}
+
+/**
+ * SQLAlchemy driver scheme options.
+ */
+export enum RedshiftScheme {
+    RedshiftPsycopg2 = "redshift+psycopg2",
+}
+
+/**
+ * Client SSL configuration
+ *
+ * OpenMetadata Client configured to validate SSL certificates.
+ */
+export interface Config {
+    /**
+     * The CA certificate used for SSL validation.
+     */
+    caCertificate?: string;
+    /**
+     * The SSL certificate used for client authentication.
+     */
+    sslCertificate?: string;
+    /**
+     * The private key associated with the SSL certificate.
+     */
+    sslKey?: string;
+}
+
+/**
+ * SSL Mode to connect to database.
+ */
+export enum SSLMode {
+    Allow = "allow",
+    Disable = "disable",
+    Prefer = "prefer",
+    Require = "require",
+    VerifyCA = "verify-ca",
+    VerifyFull = "verify-full",
+}
+
+/**
+ * Service Type
+ *
+ * Service type.
+ */
+export enum RedshiftType {
+    Redshift = "Redshift",
+}

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/metadataExporterConnectors/snowflakeConnection.ts

@@ -0,0 +1,89 @@
+/*
+ *  Copyright 2025 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+/**
+ * Snowflake Connection Config
+ */
+export interface SnowflakeConnection {
+    /**
+     * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the
+     * account is xyz1234.us-east-1.gcp
+     */
+    account: string;
+    /**
+     * Optional configuration for ingestion to keep the client session active in case the
+     * ingestion process runs for longer durations.
+     */
+    clientSessionKeepAlive?: boolean;
+    connectionArguments?:    { [key: string]: any };
+    connectionOptions?:      { [key: string]: string };
+    /**
+     * Database of the data source. This is optional parameter, if you would like to restrict
+     * the metadata reading to a single database. When left blank, OpenMetadata Ingestion
+     * attempts to scan all the databases.
+     */
+    database?: string;
+    /**
+     * Password to connect to Snowflake.
+     */
+    password?: string;
+    /**
+     * Connection to Snowflake instance via Private Key
+     */
+    privateKey?: string;
+    /**
+     * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user
+     * should have enough privileges to alter the session.
+     */
+    queryTag?: string;
+    /**
+     * Snowflake Role.
+     */
+    role?: string;
+    /**
+     * SQLAlchemy driver scheme options.
+     */
+    scheme?: SnowflakeScheme;
+    /**
+     * Snowflake Passphrase Key used with Private Key
+     */
+    snowflakePrivatekeyPassphrase?: string;
+    /**
+     * Service Type
+     */
+    type?: SnowflakeType;
+    /**
+     * Username to connect to Snowflake. This user should have privileges to read all the
+     * metadata in Snowflake.
+     */
+    username: string;
+    /**
+     * Snowflake warehouse.
+     */
+    warehouse: string;
+}
+
+/**
+ * SQLAlchemy driver scheme options.
+ */
+export enum SnowflakeScheme {
+    Snowflake = "snowflake",
+}
+
+/**
+ * Service Type
+ *
+ * Service type.
+ */
+export enum SnowflakeType {
+    Snowflake = "Snowflake",
+}

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/createAppRequest.ts

@@ -35,6 +35,11 @@ export interface CreateAppRequest {
      * Fully qualified names of the domains the Application belongs to.
      */
     domains?: string[];
+    /**
+     * The ingestion agent responsible for executing the ingestion pipeline. It will be defined
+     * at runtime based on the Ingestion Agent of the service.
+     */
+    ingestionRunner?: EntityReference;
     /**
      * Name of the Application.
      */
@@ -70,14 +75,17 @@ export enum ScheduleTimeline {
 }
 
 /**
- * Owners of this workflow.
+ * The ingestion agent responsible for executing the ingestion pipeline. It will be defined
+ * at runtime based on the Ingestion Agent of the service.
  *
- * This schema defines the EntityReferenceList type used for referencing an entity.
+ * This schema defines the EntityReference type used for referencing an entity.
  * EntityReference is used for capturing relationships from one entity to another. For
  * example, a table has an attribute called database of type EntityReference that captures
  * the relationship of a table `belongs to a` database.
  *
- * This schema defines the EntityReference type used for referencing an entity.
+ * Owners of this workflow.
+ *
+ * This schema defines the EntityReferenceList type used for referencing an entity.
  * EntityReference is used for capturing relationships from one entity to another. For
  * example, a table has an attribute called database of type EntityReference that captures
  * the relationship of a table `belongs to a` database.

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts

@@ -135,6 +135,10 @@ export interface AppMarketPlaceDefinition {
      * Support Email for the application
      */
     supportEmail?: string;
+    /**
+     * If the app support execution through the external runner.
+     */
+    supportsIngestionRunner?: boolean;
     /**
      * If the app run can be interrupted as part of the execution.
      */

openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts

@@ -109,6 +109,10 @@ export interface CreateAppMarketPlaceDefinitionReq {
      * Support Email for the application
      */
     supportEmail?: string;
+    /**
+     * If the app support execution through the external runner.
+     */
+    supportsIngestionRunner?: boolean;
     /**
      * If the app run can be interrupted as part of the execution.
      */

openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts

@@ -223,6 +223,8 @@ export interface FieldChange {
  * Reference to the data contract that this test suite is associated with.
  *
  * DEPRECATED in 1.6.2: Use 'basicEntityReference'.
+ *
+ * Link to the ingestion pipeline that ingested this entity.
  */
 export interface EntityReference {
     /**
@@ -557,6 +559,10 @@ export interface TestSuite {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * Link to the ingestion pipeline that ingested this entity.
+     */
+    ingestionRunner?: EntityReference;
     /**
      * Indicates if the test suite is inherited from a parent entity.
      */

openmetadata-ui/src/main/resources/ui/src/generated/tests/testSuite.ts

@@ -78,6 +78,10 @@ export interface TestSuite {
      * Change that lead to this version of the entity.
      */
     incrementalChangeDescription?: ChangeDescription;
+    /**
+     * Link to the ingestion pipeline that ingested this entity.
+     */
+    ingestionRunner?: EntityReference;
     /**
      * Indicates if the test suite is inherited from a parent entity.
      */
@@ -151,6 +155,8 @@ export interface TestSuite {
  * the relationship of a table `belongs to a` database.
  *
  * DEPRECATED in 1.6.2: Use 'basicEntityReference'.
+ *
+ * Link to the ingestion pipeline that ingested this entity.
  */
 export interface EntityReference {
     /**

openmetadata-ui/src/main/resources/ui/src/generated/type/entityProfile.ts

@@ -0,0 +1,311 @@
+/*
+ *  Copyright 2025 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+/**
+ * This schema defines the type to capture the full table Entity profile.
+ */
+export interface EntityProfile {
+    /**
+     * Reference to the entity for which this profile is created.
+     */
+    entityReference: EntityReference;
+    /**
+     * Unique identifier of this profile instance
+     */
+    id: string;
+    /**
+     * Profile data specific to the entity type.
+     */
+    profileData: Profile;
+    /**
+     * type of profile
+     */
+    profileType?: ProfileTypeEnum;
+    /**
+     * Data one which test case result is taken.
+     */
+    timestamp: number;
+}
+
+/**
+ * Reference to the entity for which this profile is created.
+ *
+ * This schema defines the EntityReference type used for referencing an entity.
+ * EntityReference is used for capturing relationships from one entity to another. For
+ * example, a table has an attribute called database of type EntityReference that captures
+ * the relationship of a table `belongs to a` database.
+ */
+export interface EntityReference {
+    /**
+     * If true the entity referred to has been soft-deleted.
+     */
+    deleted?: boolean;
+    /**
+     * Optional description of entity.
+     */
+    description?: string;
+    /**
+     * Display Name that identifies this entity.
+     */
+    displayName?: string;
+    /**
+     * Fully qualified name of the entity instance. For entities such as tables, databases
+     * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy
+     * such as `user` and `team` this will be same as the `name` field.
+     */
+    fullyQualifiedName?: string;
+    /**
+     * Link to the entity resource.
+     */
+    href?: string;
+    /**
+     * Unique identifier that identifies an entity instance.
+     */
+    id: string;
+    /**
+     * If true the relationship indicated by this entity reference is inherited from the parent
+     * entity.
+     */
+    inherited?: boolean;
+    /**
+     * Name of the entity instance.
+     */
+    name?: string;
+    /**
+     * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`,
+     * `dashboardService`...
+     */
+    type: string;
+}
+
+/**
+ * Profile data specific to the entity type.
+ *
+ * This schema defines the type to capture the table's data profile.
+ *
+ * This schema defines the type to capture the table's column profile.
+ *
+ * This schema defines the System Profile object holding profile data from system tables.
+ */
+export interface Profile {
+    /**
+     * No.of columns in the table.
+     */
+    columnCount?: number;
+    /**
+     * Table creation time.
+     */
+    createDateTime?: Date;
+    /**
+     * Custom Metrics profile list bound to a column.
+     */
+    customMetrics?: CustomMetricProfile[];
+    /**
+     * Percentage of data or no. of rows we want to execute the profiler and tests on
+     */
+    profileSample?:     number;
+    profileSampleType?: ProfileSampleType;
+    /**
+     * No.of rows in the table. This is always executed on the whole table.
+     */
+    rowCount?:           number;
+    samplingMethodType?: SamplingMethodType;
+    /**
+     * Table size in GB
+     */
+    sizeInByte?: number;
+    /**
+     * Timestamp on which profile is taken.
+     */
+    timestamp?: number;
+    /**
+     * Number of values that contain distinct values.
+     */
+    distinctCount?: number;
+    /**
+     * Proportion of distinct values in a column.
+     */
+    distinctProportion?: number;
+    /**
+     * No.of Rows that contain duplicates in a column.
+     */
+    duplicateCount?: number;
+    /**
+     * First quartile of a column.
+     */
+    firstQuartile?: number;
+    /**
+     * Histogram of a column.
+     */
+    histogram?: any[] | boolean | HistogramClass | number | number | null | string;
+    /**
+     * Inter quartile range of a column.
+     */
+    interQuartileRange?: number;
+    /**
+     * Maximum value in a column.
+     */
+    max?: number | string;
+    /**
+     * Maximum string length in a column.
+     */
+    maxLength?: number;
+    /**
+     * Avg value in a column.
+     */
+    mean?: number;
+    /**
+     * Median of a column.
+     */
+    median?: number;
+    /**
+     * Minimum value in a column.
+     */
+    min?: number | string;
+    /**
+     * Minimum string length in a column.
+     */
+    minLength?: number;
+    /**
+     * Missing count is calculated by subtracting valuesCount - validCount.
+     */
+    missingCount?: number;
+    /**
+     * Missing Percentage is calculated by taking percentage of validCount/valuesCount.
+     */
+    missingPercentage?: number;
+    /**
+     * Column Name.
+     */
+    name?: string;
+    /**
+     * Non parametric skew of a column.
+     */
+    nonParametricSkew?: number;
+    /**
+     * No.of null values in a column.
+     */
+    nullCount?: number;
+    /**
+     * No.of null value proportion in columns.
+     */
+    nullProportion?: number;
+    /**
+     * Standard deviation of a column.
+     */
+    stddev?: number;
+    /**
+     * Median value in a column.
+     */
+    sum?: number;
+    /**
+     * First quartile of a column.
+     */
+    thirdQuartile?: number;
+    /**
+     * No. of unique values in the column.
+     */
+    uniqueCount?: number;
+    /**
+     * Proportion of number of unique values in a column.
+     */
+    uniqueProportion?: number;
+    /**
+     * Total count of valid values in this column.
+     */
+    validCount?: number;
+    /**
+     * Total count of the values in this column.
+     */
+    valuesCount?: number;
+    /**
+     * Percentage of values in this column with respect to row count.
+     */
+    valuesPercentage?: number;
+    /**
+     * Variance of a column.
+     */
+    variance?: number;
+    /**
+     * Operation performed.
+     */
+    operation?: DMLOperationType;
+    /**
+     * Number of rows affected.
+     */
+    rowsAffected?: number;
+    [property: string]: any;
+}
+
+/**
+ * Profiling results of a Custom Metric.
+ */
+export interface CustomMetricProfile {
+    /**
+     * Custom metric name.
+     */
+    name?: string;
+    /**
+     * Profiling results for the metric.
+     */
+    value?: number;
+}
+
+export interface HistogramClass {
+    /**
+     * Boundaries of Histogram.
+     */
+    boundaries?: any[];
+    /**
+     * Frequencies of Histogram.
+     */
+    frequencies?: any[];
+}
+
+/**
+ * Operation performed.
+ *
+ * This schema defines the type of DML operation.
+ */
+export enum DMLOperationType {
+    Delete = "DELETE",
+    Insert = "INSERT",
+    Update = "UPDATE",
+    Write = "WRITE",
+}
+
+/**
+ * Type of Profile Sample (percentage or rows)
+ */
+export enum ProfileSampleType {
+    Percentage = "PERCENTAGE",
+    Rows = "ROWS",
+}
+
+/**
+ * Type of Sampling Method (BERNOULLI or SYSTEM)
+ */
+export enum SamplingMethodType {
+    Bernoulli = "BERNOULLI",
+    System = "SYSTEM",
+}
+
+/**
+ * type of profile
+ *
+ * profile type
+ */
+export enum ProfileTypeEnum {
+    Column = "column",
+    System = "system",
+    Table = "table",
+}