yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-29 17:49:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

ISSUE #2031 - Add App Runner to Application Entity (#23259)

Browse Source 
* feat: added exporter app config

* refactor: added entityprofile resource & added backward compatibility to existing API

* feat: added tests to get_profile_data_by_type

* feat: remove non supported event types

* chore: added migrations to 1.9.7

* chore: added application creation readme

* feat: implemented backend support for agent runner for external applications

* chore: move migrations to 1.9.8

* Update generated TypeScript types

* fix: added setIngestionRunner method to serviceEntityInterface interface

* Update generated TypeScript types

* fix: remove service instanceof MetadataService in deploy endpoint

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
(cherry picked from commit 568baab4267186490f88237f623cf886ce75b1b8)
This commit is contained in:
Teddy 2025-09-08 08:04:50 +02:00 committed by Teddy Crepineau
parent 14607042b6
commit 4159a2345c
22 changed files with 1290 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java
View File

@ -9,6 +9,7 @@ import javax.validation.ConstraintViolationException;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.entity.app.App;
import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition;
import org.openmetadata.schema.entity.app.AppType;
import org.openmetadata.schema.entity.app.CreateApp;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
@ -30,6 +31,9 @@ public class AppMapper implements EntityMapper<App, CreateApp> {
null,
createAppRequest.getName(),
new EntityUtil.Fields(appMarketPlaceRepository.getAllowedFields()));
Boolean supportsIngestionRunner =
!marketPlaceDefinition.getAppType().equals(AppType.Internal)
&& marketPlaceDefinition.getSupportsIngestionRunner();
List<EntityReference> owners = validateOwners(createAppRequest.getOwners());
App app =
new App()
@ -59,7 +63,12 @@ public class AppMapper implements EntityMapper<App, CreateApp> {
.withAllowConfiguration(marketPlaceDefinition.getAllowConfiguration())
.withSystem(marketPlaceDefinition.getSystem())
.withSupportsInterrupt(marketPlaceDefinition.getSupportsInterrupt())
.withFullyQualifiedName(marketPlaceDefinition.getFullyQualifiedName());
.withFullyQualifiedName(marketPlaceDefinition.getFullyQualifiedName())
.withSupportsIngestionRunner(supportsIngestionRunner)
.withIngestionRunner(
supportsIngestionRunner.equals(true)
? createAppRequest.getIngestionRunner()
: null);
// validate Bot if provided
validateAndAddBot(app, createAppRequest.getBot());

3
openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java
View File

@ -44,7 +44,8 @@ public class AppMarketPlaceMapper
.withAllowConfiguration(create.getAllowConfiguration())
.withSystem(create.getSystem())
.withSupportsInterrupt(create.getSupportsInterrupt())
.withEventSubscriptions(create.getEventSubscriptions());
.withEventSubscriptions(create.getEventSubscriptions())
.withSupportsIngestionRunner(create.getSupportsIngestionRunner());
// Validate App
validateApplication(app);

10
openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java
View File

@ -1080,6 +1080,11 @@ public class AppResource extends EntityResource<App, AppRepository> {
IngestionPipeline ingestionPipeline = getIngestionPipeline(uriInfo, securityContext, app);
ServiceEntityInterface service =
Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED);
if (app.getSupportsIngestionRunner()) {
service.setIngestionRunner(app.getIngestionRunner());
}
PipelineServiceClientResponse response =
pipelineServiceClient.runPipeline(ingestionPipeline, service, configPayload);
return Response.status(response.getCode()).entity(response).build();
@ -1165,6 +1170,11 @@ public class AppResource extends EntityResource<App, AppRepository> {
IngestionPipeline ingestionPipeline = getIngestionPipeline(uriInfo, securityContext, app);
ServiceEntityInterface service =
Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED);
if (app.getSupportsIngestionRunner()) {
service.setIngestionRunner(app.getIngestionRunner());
}
PipelineServiceClientResponse status =
pipelineServiceClient.deployPipeline(ingestionPipeline, service);
if (status.getCode() == 200) {

2
openmetadata-spec/src/main/java/org/openmetadata/schema/ServiceEntityInterface.java
View File

@ -29,6 +29,8 @@ public interface ServiceEntityInterface extends EntityInterface {
void setPipelines(List<EntityReference> pipelines);
void setIngestionRunner(EntityReference ingestionRunner);
void setTestConnectionResult(TestConnectionResult testConnectionResult);
EnumInterface getServiceType();

9
openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json
View File

@ -304,6 +304,15 @@
"eventSubscriptions": {
"description": "Event Subscriptions for the Application.",
"$ref": "../../type/entityReferenceList.json"
},
"supportsIngestionRunner": {
"description": "If the app support execution through the external runner.",
"type": "boolean",
"default": false
},
"ingestionRunner" : {
"description": "The ingestion agent responsible for executing the ingestion pipeline. It will be defined at runtime based on the Ingestion Agent of the service.",
"$ref": "../../type/entityReference.json"
}
},
"additionalProperties": false,

4
openmetadata-spec/src/main/resources/json/schema/entity/applications/createAppRequest.json
View File

@ -46,6 +46,10 @@
"description": "If the app run can be interrupted as part of the execution.",
"type": "boolean",
"default": false
},
"ingestionRunner" : {
"description": "The ingestion agent responsible for executing the ingestion pipeline. It will be defined at runtime based on the Ingestion Agent of the service.",
"$ref": "../../type/entityReference.json"
}
},
"additionalProperties": false

5
openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json
View File

@ -165,6 +165,11 @@
"items": {
"$ref": "../../../events/api/createEventSubscription.json"
}
},
"supportsIngestionRunner": {
"description": "If the app support execution through the external runner.",
"type": "boolean",
"default": false
}
},
"additionalProperties": false,

5
openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json
View File

@ -131,6 +131,11 @@
"description": "The app will be installable only if this flag is set to true.",
"type": "boolean",
"default": true
},
"supportsIngestionRunner": {
"description": "If the app support execution through the external runner.",
"type": "boolean",
"default": false
}
},
"additionalProperties": false,

4
openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json
View File

@ -175,6 +175,10 @@
"description": "Indicates if the test suite is inherited from a parent entity.",
"type": "boolean",
"default": false
},
"ingestionRunner": {
"description": "Link to the ingestion pipeline that ingested this entity.",
"$ref": "../type/entityReference.json"
}
},
"required": [

251
openmetadata-ui/src/main/resources/ui/src/generated/api/data/createEntityProfile.ts Normal file
View File

@ -0,0 +1,251 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Request schema for creating an entity profile for a table.
*/
export interface CreateEntityProfile {
/**
* Type of entity for which the profile is created. For example, 'table'.
*/
entityType: string;
profileData: Profile;
/**
* type of profile
*/
profileType?: ProfileTypeEnum;
/**
* Timestamp when the profile is created.
*/
timestamp?: number;
}
/**
* This schema defines the type to capture the table's data profile.
*
* This schema defines the type to capture the table's column profile.
*
* This schema defines the System Profile object holding profile data from system tables.
*/
export interface Profile {
/**
* No.of columns in the table.
*/
columnCount?: number;
/**
* Table creation time.
*/
createDateTime?: Date;
/**
* Custom Metrics profile list bound to a column.
*/
customMetrics?: CustomMetricProfile[];
/**
* Percentage of data or no. of rows we want to execute the profiler and tests on
*/
profileSample?: number;
profileSampleType?: ProfileSampleType;
/**
* No.of rows in the table. This is always executed on the whole table.
*/
rowCount?: number;
samplingMethodType?: SamplingMethodType;
/**
* Table size in GB
*/
sizeInByte?: number;
/**
* Timestamp on which profile is taken.
*/
timestamp?: number;
/**
* Number of values that contain distinct values.
*/
distinctCount?: number;
/**
* Proportion of distinct values in a column.
*/
distinctProportion?: number;
/**
* No.of Rows that contain duplicates in a column.
*/
duplicateCount?: number;
/**
* First quartile of a column.
*/
firstQuartile?: number;
/**
* Histogram of a column.
*/
histogram?: any[] | boolean | HistogramClass | number | number | null | string;
/**
* Inter quartile range of a column.
*/
interQuartileRange?: number;
/**
* Maximum value in a column.
*/
max?: number | string;
/**
* Maximum string length in a column.
*/
maxLength?: number;
/**
* Avg value in a column.
*/
mean?: number;
/**
* Median of a column.
*/
median?: number;
/**
* Minimum value in a column.
*/
min?: number | string;
/**
* Minimum string length in a column.
*/
minLength?: number;
/**
* Missing count is calculated by subtracting valuesCount - validCount.
*/
missingCount?: number;
/**
* Missing Percentage is calculated by taking percentage of validCount/valuesCount.
*/
missingPercentage?: number;
/**
* Column Name.
*/
name?: string;
/**
* Non parametric skew of a column.
*/
nonParametricSkew?: number;
/**
* No.of null values in a column.
*/
nullCount?: number;
/**
* No.of null value proportion in columns.
*/
nullProportion?: number;
/**
* Standard deviation of a column.
*/
stddev?: number;
/**
* Median value in a column.
*/
sum?: number;
/**
* First quartile of a column.
*/
thirdQuartile?: number;
/**
* No. of unique values in the column.
*/
uniqueCount?: number;
/**
* Proportion of number of unique values in a column.
*/
uniqueProportion?: number;
/**
* Total count of valid values in this column.
*/
validCount?: number;
/**
* Total count of the values in this column.
*/
valuesCount?: number;
/**
* Percentage of values in this column with respect to row count.
*/
valuesPercentage?: number;
/**
* Variance of a column.
*/
variance?: number;
/**
* Operation performed.
*/
operation?: DMLOperationType;
/**
* Number of rows affected.
*/
rowsAffected?: number;
[property: string]: any;
}
/**
* Profiling results of a Custom Metric.
*/
export interface CustomMetricProfile {
/**
* Custom metric name.
*/
name?: string;
/**
* Profiling results for the metric.
*/
value?: number;
}
export interface HistogramClass {
/**
* Boundaries of Histogram.
*/
boundaries?: any[];
/**
* Frequencies of Histogram.
*/
frequencies?: any[];
}
/**
* Operation performed.
*
* This schema defines the type of DML operation.
*/
export enum DMLOperationType {
Delete = "DELETE",
Insert = "INSERT",
Update = "UPDATE",
Write = "WRITE",
}
/**
* Type of Profile Sample (percentage or rows)
*/
export enum ProfileSampleType {
Percentage = "PERCENTAGE",
Rows = "ROWS",
}
/**
* Type of Sampling Method (BERNOULLI or SYSTEM)
*/
export enum SamplingMethodType {
Bernoulli = "BERNOULLI",
System = "SYSTEM",
}
/**
* type of profile
*
* profile type
*/
export enum ProfileTypeEnum {
Column = "column",
System = "system",
Table = "table",
}

15
openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts
View File

@ -104,6 +104,11 @@ export interface App {
* Change that lead to this version of the entity.
*/
incrementalChangeDescription?: ChangeDescription;
/**
* The ingestion agent responsible for executing the ingestion pipeline. It will be defined
* at runtime based on the Ingestion Agent of the service.
*/
ingestionRunner?: EntityReference;
/**
* Name of the Application.
*/
@ -153,6 +158,10 @@ export interface App {
* Support Email for the application
*/
supportEmail?: string;
/**
* If the app support execution through the external runner.
*/
supportsIngestionRunner?: boolean;
/**
* If the app run can be interrupted as part of the execution.
*/
@ -547,6 +556,9 @@ export interface Action {
* the relationship of a table `belongs to a` database.
*
* Bot User Associated with this application.
*
* The ingestion agent responsible for executing the ingestion pipeline. It will be defined
* at runtime based on the Ingestion Agent of the service.
*/
export interface EntityReference {
/**
@ -646,6 +658,9 @@ export enum MetadataAttribute {
* the relationship of a table `belongs to a` database.
*
* Bot User Associated with this application.
*
* The ingestion agent responsible for executing the ingestion pipeline. It will be defined
* at runtime based on the Ingestion Agent of the service.
*/
export interface TagLabel {
/**

186
openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/metadataExporterAppConfig.ts vendored Normal file
View File

@ -0,0 +1,186 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Configuration for the Metadata Exporter Application.
*/
export interface MetadataExporterAppConfig {
/**
* Enable backfill for the exporter to process historical data. This will only work on the
* very first run of the exporter.
*/
backfill?: boolean;
/**
* Connection details for the Metadata Exporter Application.
*/
connectionConfig: SnowflakeConnection;
/**
* List of event types to export.
*/
eventTypes?: EventType[];
/**
* Range of data to export. Options are 'ALL' for all data, 'LATEST' for the latest data, or
* a specific date range.
*/
exportRange: ExportRangeConfiguration;
/**
* Configuration for the table to export the data to.
*/
tableConfiguration: ConfigurationForTheTableToExportTheDataTo;
/**
* Application Type
*/
type?: MetadataExporterAppType;
}
/**
* Connection details for the Metadata Exporter Application.
*
* Snowflake Connection Config
*/
export interface SnowflakeConnection {
/**
* If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the
* account is xyz1234.us-east-1.gcp
*/
account: string;
/**
* Optional configuration for ingestion to keep the client session active in case the
* ingestion process runs for longer durations.
*/
clientSessionKeepAlive?: boolean;
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Database of the data source. This is optional parameter, if you would like to restrict
* the metadata reading to a single database. When left blank, OpenMetadata Ingestion
* attempts to scan all the databases.
*/
database?: string;
/**
* Password to connect to Snowflake.
*/
password?: string;
/**
* Connection to Snowflake instance via Private Key
*/
privateKey?: string;
/**
* Session query tag used to monitor usage on snowflake. To use a query tag snowflake user
* should have enough privileges to alter the session.
*/
queryTag?: string;
/**
* Snowflake Role.
*/
role?: string;
/**
* SQLAlchemy driver scheme options.
*/
scheme?: SnowflakeScheme;
/**
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Service Type
*/
type?: SnowflakeType;
/**
* Username to connect to Snowflake. This user should have privileges to read all the
* metadata in Snowflake.
*/
username: string;
/**
* Snowflake warehouse.
*/
warehouse: string;
}
/**
* SQLAlchemy driver scheme options.
*/
export enum SnowflakeScheme {
Snowflake = "snowflake",
}
/**
* Service Type
*
* Service type.
*/
export enum SnowflakeType {
Snowflake = "Snowflake",
}
export enum EventType {
Profile = "PROFILE",
TestCaseResults = "TEST_CASE_RESULTS",
}
/**
* Range of data to export. Options are 'ALL' for all data, 'LATEST' for the latest data, or
* a specific date range.
*
* Configuration for the export range of data.
*/
export interface ExportRangeConfiguration {
/**
* Interval for the export range, applicable when rangeType is 'DATE_RANGE'.
*/
interval?: number;
/**
* Type of range for data export.
*/
rangeType: RangeType;
/**
* Unit of time for the export range.
*/
unit?: Unit;
[property: string]: any;
}
/**
* Type of range for data export.
*/
export enum RangeType {
All = "ALL",
DateRange = "DATE_RANGE",
Latest = "LATEST",
}
/**
* Unit of time for the export range.
*/
export enum Unit {
Days = "DAYS",
Hours = "HOURS",
}
/**
* Configuration for the table to export the data to.
*/
export interface ConfigurationForTheTableToExportTheDataTo {
databaseName: string;
schemaName: string;
tableName: string;
[property: string]: any;
}
/**
* Application Type
*
* Application type.
*/
export enum MetadataExporterAppType {
MetadataExporter = "MetadataExporter",
}

188
openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/metadataExporterConnectors/bigQueryConnection.ts vendored Normal file
View File

@ -0,0 +1,188 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Google BigQuery Connection Config
*/
export interface BigQueryConnection {
/**
* Billing Project ID
*/
billingProjectId?: string;
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* GCP Credentials
*/
credentials: GCPCredentials;
/**
* BigQuery APIs URL.
*/
hostPort?: string;
/**
* SQLAlchemy driver scheme options.
*/
scheme?: BigqueryScheme;
/**
* Taxonomy location used to fetch policy tags
*/
taxonomyLocation?: string;
/**
* Project IDs used to fetch policy tags
*/
taxonomyProjectID?: string[];
/**
* Service Type
*/
type?: BigqueryType;
/**
* Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can
* pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia
* multi-regions are not yet in GA.
*/
usageLocation?: string;
}
/**
* GCP Credentials
*
* GCP credentials configs.
*/
export interface GCPCredentials {
/**
* We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP
* Credentials Path
*/
gcpConfig: GCPCredentialsConfiguration;
/**
* we enable the authenticated service account to impersonate another service account
*/
gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues;
}
/**
* We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP
* Credentials Path
*
* Pass the raw credential values provided by GCP
*
* Pass the path of file containing the GCP credentials info
*
* Use the application default credentials
*/
export interface GCPCredentialsConfiguration {
/**
* Google Cloud auth provider certificate.
*/
authProviderX509CertUrl?: string;
/**
* Google Cloud auth uri.
*/
authUri?: string;
/**
* Google Cloud email.
*/
clientEmail?: string;
/**
* Google Cloud Client ID.
*/
clientId?: string;
/**
* Google Cloud client certificate uri.
*/
clientX509CertUrl?: string;
/**
* Google Cloud private key.
*/
privateKey?: string;
/**
* Google Cloud private key id.
*/
privateKeyId?: string;
/**
* Project ID
*
* GCP Project ID to parse metadata from
*/
projectId?: string[] | string;
/**
* Google Cloud token uri.
*/
tokenUri?: string;
/**
* Google Cloud Platform account type.
*
* Google Cloud Platform ADC ( Application Default Credentials )
*/
type?: string;
/**
* Path of the file containing the GCP credentials info
*/
path?: string;
/**
* Google Security Token Service audience which contains the resource name for the workload
* identity pool and the provider identifier in that pool.
*/
audience?: string;
/**
* This object defines the mechanism used to retrieve the external credential from the local
* environment so that it can be exchanged for a GCP access token via the STS endpoint
*/
credentialSource?: { [key: string]: string };
/**
* Google Cloud Platform account type.
*/
externalType?: string;
/**
* Google Security Token Service subject token type based on the OAuth 2.0 token exchange
* spec.
*/
subjectTokenType?: string;
/**
* Google Security Token Service token exchange endpoint.
*/
tokenURL?: string;
[property: string]: any;
}
/**
* we enable the authenticated service account to impersonate another service account
*
* Pass the values to impersonate a service account of Google Cloud
*/
export interface GCPImpersonateServiceAccountValues {
/**
* The impersonated service account email
*/
impersonateServiceAccount?: string;
/**
* Number of seconds the delegated credential should be valid
*/
lifetime?: number;
[property: string]: any;
}
/**
* SQLAlchemy driver scheme options.
*/
export enum BigqueryScheme {
Bigquery = "bigquery",
}
/**
* Service Type
*
* Service type.
*/
export enum BigqueryType {
BigQuery = "BigQuery",
}

73
openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/metadataExporterConnectors/databricksConnection.ts vendored Normal file
View File

@ -0,0 +1,73 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Databricks Connection Config
*/
export interface DatabricksConnection {
/**
* Catalog of the data source(Example: hive_metastore). This is optional parameter, if you
* would like to restrict the metadata reading to a single catalog. When left blank,
* OpenMetadata Ingestion attempts to scan all the catalog.
*/
catalog?: string;
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* The maximum amount of time (in seconds) to wait for a successful connection to the data
* source. If the connection attempt takes longer than this timeout period, an error will be
* returned.
*/
connectionTimeout?: number;
/**
* Database Schema of the data source. This is optional parameter, if you would like to
* restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion
* attempts to scan all the schemas.
*/
databaseSchema?: string;
/**
* Host and port of the Databricks service.
*/
hostPort: string;
/**
* Databricks compute resources URL.
*/
httpPath: string;
/**
* SQLAlchemy driver scheme options.
*/
scheme?: DatabricksScheme;
/**
* Generated Token to connect to Databricks.
*/
token: string;
/**
* Service Type
*/
type?: DatabricksType;
}
/**
* SQLAlchemy driver scheme options.
*/
export enum DatabricksScheme {
DatabricksConnector = "databricks+connector",
}
/**
* Service Type
*
* Service type.
*/
export enum DatabricksType {
Databricks = "Databricks",
}

95
openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/metadataExporterConnectors/redshiftConnection.ts vendored Normal file
View File

@ -0,0 +1,95 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Redshift Connection Config
*/
export interface RedshiftConnection {
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Initial Redshift database to connect to. If you want to ingest all databases, set
* ingestAllDatabases to true.
*/
database: string;
/**
* Host and port of the Redshift service.
*/
hostPort: string;
/**
* Password to connect to Redshift.
*/
password?: string;
/**
* SQLAlchemy driver scheme options.
*/
scheme?: RedshiftScheme;
sslConfig?: Config;
sslMode?: SSLMode;
/**
* Service Type
*/
type?: RedshiftType;
/**
* Username to connect to Redshift. This user should have privileges to read all the
* metadata in Redshift.
*/
username: string;
}
/**
* SQLAlchemy driver scheme options.
*/
export enum RedshiftScheme {
RedshiftPsycopg2 = "redshift+psycopg2",
}
/**
* Client SSL configuration
*
* OpenMetadata Client configured to validate SSL certificates.
*/
export interface Config {
/**
* The CA certificate used for SSL validation.
*/
caCertificate?: string;
/**
* The SSL certificate used for client authentication.
*/
sslCertificate?: string;
/**
* The private key associated with the SSL certificate.
*/
sslKey?: string;
}
/**
* SSL Mode to connect to database.
*/
export enum SSLMode {
Allow = "allow",
Disable = "disable",
Prefer = "prefer",
Require = "require",
VerifyCA = "verify-ca",
VerifyFull = "verify-full",
}
/**
* Service Type
*
* Service type.
*/
export enum RedshiftType {
Redshift = "Redshift",
}

89
openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/metadataExporterConnectors/snowflakeConnection.ts vendored Normal file
View File

@ -0,0 +1,89 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Snowflake Connection Config
*/
export interface SnowflakeConnection {
/**
* If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the
* account is xyz1234.us-east-1.gcp
*/
account: string;
/**
* Optional configuration for ingestion to keep the client session active in case the
* ingestion process runs for longer durations.
*/
clientSessionKeepAlive?: boolean;
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Database of the data source. This is optional parameter, if you would like to restrict
* the metadata reading to a single database. When left blank, OpenMetadata Ingestion
* attempts to scan all the databases.
*/
database?: string;
/**
* Password to connect to Snowflake.
*/
password?: string;
/**
* Connection to Snowflake instance via Private Key
*/
privateKey?: string;
/**
* Session query tag used to monitor usage on snowflake. To use a query tag snowflake user
* should have enough privileges to alter the session.
*/
queryTag?: string;
/**
* Snowflake Role.
*/
role?: string;
/**
* SQLAlchemy driver scheme options.
*/
scheme?: SnowflakeScheme;
/**
* Snowflake Passphrase Key used with Private Key
*/
snowflakePrivatekeyPassphrase?: string;
/**
* Service Type
*/
type?: SnowflakeType;
/**
* Username to connect to Snowflake. This user should have privileges to read all the
* metadata in Snowflake.
*/
username: string;
/**
* Snowflake warehouse.
*/
warehouse: string;
}
/**
* SQLAlchemy driver scheme options.
*/
export enum SnowflakeScheme {
Snowflake = "snowflake",
}
/**
* Service Type
*
* Service type.
*/
export enum SnowflakeType {
Snowflake = "Snowflake",
}

14
openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/createAppRequest.ts
View File

@ -35,6 +35,11 @@ export interface CreateAppRequest {
* Fully qualified names of the domains the Application belongs to.
*/
domains?: string[];
/**
* The ingestion agent responsible for executing the ingestion pipeline. It will be defined
* at runtime based on the Ingestion Agent of the service.
*/
ingestionRunner?: EntityReference;
/**
* Name of the Application.
*/
@ -70,14 +75,17 @@ export enum ScheduleTimeline {
}
/**
* Owners of this workflow.
* The ingestion agent responsible for executing the ingestion pipeline. It will be defined
* at runtime based on the Ingestion Agent of the service.
*
* This schema defines the EntityReferenceList type used for referencing an entity.
* This schema defines the EntityReference type used for referencing an entity.
* EntityReference is used for capturing relationships from one entity to another. For
* example, a table has an attribute called database of type EntityReference that captures
* the relationship of a table `belongs to a` database.
*
* This schema defines the EntityReference type used for referencing an entity.
* Owners of this workflow.
*
* This schema defines the EntityReferenceList type used for referencing an entity.
* EntityReference is used for capturing relationships from one entity to another. For
* example, a table has an attribute called database of type EntityReference that captures
* the relationship of a table `belongs to a` database.

4
openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts
View File

@ -135,6 +135,10 @@ export interface AppMarketPlaceDefinition {
* Support Email for the application
*/
supportEmail?: string;
/**
* If the app support execution through the external runner.
*/
supportsIngestionRunner?: boolean;
/**
* If the app run can be interrupted as part of the execution.
*/

4
openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts
View File

@ -109,6 +109,10 @@ export interface CreateAppMarketPlaceDefinitionReq {
* Support Email for the application
*/
supportEmail?: string;
/**
* If the app support execution through the external runner.
*/
supportsIngestionRunner?: boolean;
/**
* If the app run can be interrupted as part of the execution.
*/

6
openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts
View File

@ -223,6 +223,8 @@ export interface FieldChange {
* Reference to the data contract that this test suite is associated with.
*
* DEPRECATED in 1.6.2: Use 'basicEntityReference'.
*
* Link to the ingestion pipeline that ingested this entity.
*/
export interface EntityReference {
/**
@ -557,6 +559,10 @@ export interface TestSuite {
* Change that lead to this version of the entity.
*/
incrementalChangeDescription?: ChangeDescription;
/**
* Link to the ingestion pipeline that ingested this entity.
*/
ingestionRunner?: EntityReference;
/**
* Indicates if the test suite is inherited from a parent entity.
*/

6
openmetadata-ui/src/main/resources/ui/src/generated/tests/testSuite.ts
View File

@ -78,6 +78,10 @@ export interface TestSuite {
* Change that lead to this version of the entity.
*/
incrementalChangeDescription?: ChangeDescription;
/**
* Link to the ingestion pipeline that ingested this entity.
*/
ingestionRunner?: EntityReference;
/**
* Indicates if the test suite is inherited from a parent entity.
*/
@ -151,6 +155,8 @@ export interface TestSuite {
* the relationship of a table `belongs to a` database.
*
* DEPRECATED in 1.6.2: Use 'basicEntityReference'.
*
* Link to the ingestion pipeline that ingested this entity.
*/
export interface EntityReference {
/**

311
openmetadata-ui/src/main/resources/ui/src/generated/type/entityProfile.ts Normal file
View File

@ -0,0 +1,311 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* This schema defines the type to capture the full table Entity profile.
*/
export interface EntityProfile {
/**
* Reference to the entity for which this profile is created.
*/
entityReference: EntityReference;
/**
* Unique identifier of this profile instance
*/
id: string;
/**
* Profile data specific to the entity type.
*/
profileData: Profile;
/**
* type of profile
*/
profileType?: ProfileTypeEnum;
/**
* Data one which test case result is taken.
*/
timestamp: number;
}
/**
* Reference to the entity for which this profile is created.
*
* This schema defines the EntityReference type used for referencing an entity.
* EntityReference is used for capturing relationships from one entity to another. For
* example, a table has an attribute called database of type EntityReference that captures
* the relationship of a table `belongs to a` database.
*/
export interface EntityReference {
/**
* If true the entity referred to has been soft-deleted.
*/
deleted?: boolean;
/**
* Optional description of entity.
*/
description?: string;
/**
* Display Name that identifies this entity.
*/
displayName?: string;
/**
* Fully qualified name of the entity instance. For entities such as tables, databases
* fullyQualifiedName is returned in this field. For entities that don't have name hierarchy
* such as `user` and `team` this will be same as the `name` field.
*/
fullyQualifiedName?: string;
/**
* Link to the entity resource.
*/
href?: string;
/**
* Unique identifier that identifies an entity instance.
*/
id: string;
/**
* If true the relationship indicated by this entity reference is inherited from the parent
* entity.
*/
inherited?: boolean;
/**
* Name of the entity instance.
*/
name?: string;
/**
* Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`,
* `dashboardService`...
*/
type: string;
}
/**
* Profile data specific to the entity type.
*
* This schema defines the type to capture the table's data profile.
*
* This schema defines the type to capture the table's column profile.
*
* This schema defines the System Profile object holding profile data from system tables.
*/
export interface Profile {
/**
* No.of columns in the table.
*/
columnCount?: number;
/**
* Table creation time.
*/
createDateTime?: Date;
/**
* Custom Metrics profile list bound to a column.
*/
customMetrics?: CustomMetricProfile[];
/**
* Percentage of data or no. of rows we want to execute the profiler and tests on
*/
profileSample?: number;
profileSampleType?: ProfileSampleType;
/**
* No.of rows in the table. This is always executed on the whole table.
*/
rowCount?: number;
samplingMethodType?: SamplingMethodType;
/**
* Table size in GB
*/
sizeInByte?: number;
/**
* Timestamp on which profile is taken.
*/
timestamp?: number;
/**
* Number of values that contain distinct values.
*/
distinctCount?: number;
/**
* Proportion of distinct values in a column.
*/
distinctProportion?: number;
/**
* No.of Rows that contain duplicates in a column.
*/
duplicateCount?: number;
/**
* First quartile of a column.
*/
firstQuartile?: number;
/**
* Histogram of a column.
*/
histogram?: any[] | boolean | HistogramClass | number | number | null | string;
/**
* Inter quartile range of a column.
*/
interQuartileRange?: number;
/**
* Maximum value in a column.
*/
max?: number | string;
/**
* Maximum string length in a column.
*/
maxLength?: number;
/**
* Avg value in a column.
*/
mean?: number;
/**
* Median of a column.
*/
median?: number;
/**
* Minimum value in a column.
*/
min?: number | string;
/**
* Minimum string length in a column.
*/
minLength?: number;
/**
* Missing count is calculated by subtracting valuesCount - validCount.
*/
missingCount?: number;
/**
* Missing Percentage is calculated by taking percentage of validCount/valuesCount.
*/
missingPercentage?: number;
/**
* Column Name.
*/
name?: string;
/**
* Non parametric skew of a column.
*/
nonParametricSkew?: number;
/**
* No.of null values in a column.
*/
nullCount?: number;
/**
* No.of null value proportion in columns.
*/
nullProportion?: number;
/**
* Standard deviation of a column.
*/
stddev?: number;
/**
* Median value in a column.
*/
sum?: number;
/**
* First quartile of a column.
*/
thirdQuartile?: number;
/**
* No. of unique values in the column.
*/
uniqueCount?: number;
/**
* Proportion of number of unique values in a column.
*/
uniqueProportion?: number;
/**
* Total count of valid values in this column.
*/
validCount?: number;
/**
* Total count of the values in this column.
*/
valuesCount?: number;
/**
* Percentage of values in this column with respect to row count.
*/
valuesPercentage?: number;
/**
* Variance of a column.
*/
variance?: number;
/**
* Operation performed.
*/
operation?: DMLOperationType;
/**
* Number of rows affected.
*/
rowsAffected?: number;
[property: string]: any;
}
/**
* Profiling results of a Custom Metric.
*/
export interface CustomMetricProfile {
/**
* Custom metric name.
*/
name?: string;
/**
* Profiling results for the metric.
*/
value?: number;
}
export interface HistogramClass {
/**
* Boundaries of Histogram.
*/
boundaries?: any[];
/**
* Frequencies of Histogram.
*/
frequencies?: any[];
}
/**
* Operation performed.
*
* This schema defines the type of DML operation.
*/
export enum DMLOperationType {
Delete = "DELETE",
Insert = "INSERT",
Update = "UPDATE",
Write = "WRITE",
}
/**
* Type of Profile Sample (percentage or rows)
*/
export enum ProfileSampleType {
Percentage = "PERCENTAGE",
Rows = "ROWS",
}
/**
* Type of Sampling Method (BERNOULLI or SYSTEM)
*/
export enum SamplingMethodType {
Bernoulli = "BERNOULLI",
System = "SYSTEM",
}
/**
* type of profile
*
* profile type
*/
export enum ProfileTypeEnum {
Column = "column",
System = "system",
Table = "table",
}