diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index 8e37659781e..e1e5650851b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -202,6 +202,8 @@ public class OpenMetadataApplication extends Application; -} - -/** - * The metadata attribute to check for stopping propagation - */ -export enum MetadataAttribute { - Description = "description", - Domains = "domains", - GlossaryTerms = "glossaryTerms", - Owner = "owner", - Tags = "tags", - Tier = "tier", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - * - * Owner of this Ingestion Pipeline. - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * The processing engine responsible for executing the ingestion pipeline logic. - * - * Link to the service for which ingestion pipeline is ingesting the metadata. - * - * Service to be modified - */ -export interface TagLabel { - /** - * Description for the tag label. - * - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this tag. - * - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Link to the tag resource. - * - * Link to the entity resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType?: LabelTypeEnum; - /** - * Name of the tag or glossary term. - * - * Name of the entity instance. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source?: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state?: State; - style?: Style; - tagFQN?: string; - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id?: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type?: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelTypeEnum { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * Cover image configuration for the entity. - */ - coverImage?: CoverImage; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * Cover image configuration for the entity. - * - * Cover image configuration for an entity. This is used to display a banner or header image - * for entities like Domain, Glossary, Data Product, etc. - */ -export interface CoverImage { - /** - * Position of the cover image in CSS background-position format. Supports keywords (top, - * center, bottom) or pixel values (e.g., '20px 30px'). - */ - position?: string; - /** - * URL of the cover image. - */ - url?: string; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - */ -export interface TierElement { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelTypeEnum; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Minimum set of requirements to get a Test Case request ready - */ -export interface TestCaseDefinitions { - /** - * Compute the passed and failed row count for the test case. - */ - computePassedFailedRowCount?: boolean; - parameterValues?: TestCaseParameterValue[]; - /** - * Tags to apply - */ - tags?: TierElement[]; - /** - * Fully qualified name of the test definition. - */ - testDefinition?: string; - /** - * If the test definition supports it, use dynamic assertion to evaluate the test case. - */ - useDynamicAssertion?: boolean; - [property: string]: any; -} - -/** - * This schema defines the parameter values that can be passed for a Test Case. - */ -export interface TestCaseParameterValue { - /** - * name of the parameter. Must match the parameter names in testCaseParameterDefinition - */ - name?: string; - /** - * value to be passed for the Parameters. These are input from Users. We capture this in - * string and convert during the runtime. - */ - value?: string; - [property: string]: any; -} - -/** - * Application Type - * - * Add Tags action type. - * - * Remove Classification Tags Action Type. - * - * Add Terms action type. - * - * Remove Terms Action Type. - * - * Add Domain Action Type. - * - * Remove Domain Action Type - * - * Add Description Action Type. - * - * Add Custom Properties Action Type. - * - * Remove Description Action Type - * - * Add Tier Action Type. - * - * Remove Tier Action Type - * - * Add Test Case Action Type. - * - * Remove Test Case Action Type - * - * Add Owner Action Type. - * - * Remove Owner Action Type - * - * Remove Custom Properties Action Type. - * - * Add Data Products Action Type. - * - * Remove Data Products Action Type. - * - * Lineage propagation action type. - * - * ML PII Tagging action type. - */ -export enum ActionType { - AddCustomPropertiesAction = "AddCustomPropertiesAction", - AddDataProductAction = "AddDataProductAction", - AddDescriptionAction = "AddDescriptionAction", - AddDomainAction = "AddDomainAction", - AddOwnerAction = "AddOwnerAction", - AddTagsAction = "AddTagsAction", - AddTermsAction = "AddTermsAction", - AddTestCaseAction = "AddTestCaseAction", - AddTierAction = "AddTierAction", - LineagePropagationAction = "LineagePropagationAction", - MLTaggingAction = "MLTaggingAction", - RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", - RemoveDataProductAction = "RemoveDataProductAction", - RemoveDescriptionAction = "RemoveDescriptionAction", - RemoveDomainAction = "RemoveDomainAction", - RemoveOwnerAction = "RemoveOwnerAction", - RemoveTagsAction = "RemoveTagsAction", - RemoveTermsAction = "RemoveTermsAction", - RemoveTestCaseAction = "RemoveTestCaseAction", - RemoveTierAction = "RemoveTierAction", -} - -/** - * Backfill Configuration - */ -export interface BackfillConfiguration { - /** - * Enable Backfill for the configured dates - */ - enabled?: boolean; - /** - * Date for which the backfill will end - */ - endDate?: Date; - /** - * Date from which to start the backfill - */ - startDate?: Date; - [property: string]: any; -} - -/** - * Different Module Configurations - */ -export interface ModuleConfiguration { - /** - * App Analytics Module configuration - */ - appAnalytics: AppAnalyticsConfig; - /** - * Cost Analysis Insights Module configuration - */ - costAnalysis: CostAnalysisConfig; - /** - * Data Assets Insights Module configuration - */ - dataAssets: DataAssetsConfig; - /** - * Data Quality Insights Module configuration - */ - dataQuality: DataQualityConfig; -} - -/** - * App Analytics Module configuration - */ -export interface AppAnalyticsConfig { - /** - * If Enabled, App Analytics insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Cost Analysis Insights Module configuration - */ -export interface CostAnalysisConfig { - /** - * If Enabled, Cost Analysis insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Data Assets Insights Module configuration - */ -export interface DataAssetsConfig { - /** - * If Enabled, Data Asset insights will be populated when the App runs. - */ - enabled: boolean; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Defines the number of days the Data Assets Insights information will be kept. After it - * they will be deleted. - */ - retention?: number; - serviceFilter?: ServiceFilter; -} - -export interface ServiceFilter { - serviceName?: string; - serviceType?: string; -} - -/** - * Data Quality Insights Module configuration - */ -export interface DataQualityConfig { - /** - * If Enabled, Data Quality insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Entities selected to run the automation. - */ -export interface Resource { - /** - * Filter JSON tree to be used for rendering the filters in the UI. This comes from - * Immutable Tree type of react-awesome-query-builder. - */ - filterJsonTree?: string; - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - queryFilter?: string; - /** - * Type of the entity. E.g., 'table', 'chart',... - */ - type?: string[]; - [property: string]: any; -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Application Type - * - * Application type. - */ -export enum CollateAIAppConfigType { - AutoPilotApplication = "AutoPilotApplication", - Automator = "Automator", - CacheWarmup = "CacheWarmup", - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - CollateAITierAgent = "CollateAITierAgent", - DataInsights = "DataInsights", - DataInsightsReport = "DataInsightsReport", - SearchIndexing = "SearchIndexing", -} - -/** - * Application private configuration - * - * Private Configuration for the CollateAI External Application. - */ -export interface PrivateConfig { - /** - * Collate Server public URL. WAII will use this information to interact with the server. - * E.g., https://sandbox.getcollate.io - */ - collateURL?: string; - /** - * Limits for the CollateAI Application. - */ - limits?: AppLimitsConfig; - /** - * WAII API Token - */ - token?: string; - /** - * WAII API host URL - */ - waiiInstance?: string; - [property: string]: any; -} - -/** - * Limits for the CollateAI Application. - * - * Private Configuration for the App Limits. - */ -export interface AppLimitsConfig { - /** - * The records of the limits. - */ - actions: { [key: string]: number }; - /** - * The start of this limit cycle. - */ - billingCycleStart: Date; -} - -/** - * Available sources to fetch DBT catalog and manifest files. - * - * dbt Cloud configuration. - * - * DBT Catalog, Manifest and Run Results file path config. - * - * DBT Catalog, Manifest and Run Results HTTP path configuration. - * - * DBT Catalog, Manifest and Run Results files in S3 bucket. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in GCS storage. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in Azure bucket. We will search for - * catalog.json, manifest.json and run_results.json. - */ -export interface DBTConfigurationSource { - /** - * dbt cloud account Id - */ - dbtCloudAccountId?: string; - /** - * dbt cloud account authentication token - */ - dbtCloudAuthToken?: string; - /** - * dbt cloud job id. - */ - dbtCloudJobId?: string; - /** - * In case of multiple projects in a dbt cloud account, specify the project's id from which - * you want to extract the dbt run artifacts - */ - dbtCloudProjectId?: string; - /** - * URL to connect to your dbt cloud instance. E.g., https://cloud.getdbt.com or - * https://emea.dbt.com/ - */ - dbtCloudUrl?: string; - /** - * dbt Configuration type - */ - dbtConfigType: DbtConfigType; - /** - * DBT catalog file path to extract dbt models with their column schemas. - */ - dbtCatalogFilePath?: string; - /** - * DBT manifest file path to extract dbt models and associate with tables. - */ - dbtManifestFilePath?: string; - /** - * DBT run results file path to extract the test results information. - */ - dbtRunResultsFilePath?: string; - /** - * DBT sources file path to extract the freshness test result. - */ - dbtSourcesFilePath?: string; - /** - * DBT catalog http file path to extract dbt models with their column schemas. - */ - dbtCatalogHttpPath?: string; - /** - * DBT manifest http file path to extract dbt models and associate with tables. - */ - dbtManifestHttpPath?: string; - /** - * DBT run results http file path to extract the test results information. - */ - dbtRunResultsHttpPath?: string; - /** - * DBT sources http file path to extract freshness test results information. - */ - dbtSourcesHttpPath?: string; - /** - * Details of the bucket where the dbt files are stored - */ - dbtPrefixConfig?: DBTPrefixConfig; - dbtSecurityConfig?: Credentials; -} - -/** - * dbt Configuration type - */ -export enum DbtConfigType { - Azure = "azure", - Cloud = "cloud", - Gcs = "gcs", - HTTP = "http", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the dbt files are stored - */ -export interface DBTPrefixConfig { - /** - * Name of the bucket where the dbt files are stored - */ - dbtBucketName?: string; - /** - * Path of the folder where the dbt files are stored - */ - dbtObjectPrefix?: string; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - * - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Use incremental Metadata extraction after the first execution. This is commonly done by - * getting the changes from Audit tables on the supporting databases. - */ -export interface IncrementalMetadataExtractionConfiguration { - /** - * If True, enables Metadata Extraction to be incremental - */ - enabled: boolean; - /** - * Number os days to search back for a successful pipeline run. The timestamp of the last - * found successful pipeline run will be used as a base to search for updated entities. - */ - lookbackDays?: number; - /** - * Number of days to add to the last successful pipeline run timestamp to search for updated - * entities. - */ - safetyMarginDays?: number; -} - -/** - * Details required to generate Lineage - */ -export interface LineageInformation { - /** - * List of service path prefixes for lineage matching. Supported formats: DBServiceName, - * DBServiceName.DatabaseName, DBServiceName.DatabaseName.SchemaName, or - * DBServiceName.DatabaseName.SchemaName.TableName - */ - dbServicePrefixes?: string[]; - /** - * List of Database Service Names for creation of lineage - */ - dbServiceNames?: string[]; - /** - * List of Storage Service Names for creation of lineage - */ - storageServiceNames?: string[]; - [property: string]: any; -} - -/** - * Operation to be performed on the entity - */ -export interface Operation { - /** - * Entity to be modified - */ - entityLink: string; - /** - * The id of the operation - */ - id: string; - /** - * The configuration for the operation to be applied - */ - parameters: ReverseIngestionConfig; - /** - * Templated SQL command to be used for the operation. Context parameters will be populated - * based on the event type. - */ - SQLTemplate?: string; - /** - * Type of operation to perform - */ - type: OperationType; -} - -/** - * The configuration for the operation to be applied - * - * Configuration for updating descriptions - * - * Configuration for updating owners - * - * Configuration for updating tags - */ -export interface ReverseIngestionConfig { - /** - * New description of the service - */ - newDescription?: string; - /** - * Previous description of the service - */ - previousDescription?: string; - /** - * Added owners to be applied - */ - addedOwners?: EntityReference[]; - /** - * Removed owners from the entity - */ - removedOwners?: EntityReference[]; - /** - * Added tags to be applied - */ - addedTags?: TierElement[]; - /** - * Removed tags of the entity - */ - removedTags?: TierElement[]; -} - -/** - * Type of operation to perform - */ -export enum OperationType { - UpdateDescription = "UPDATE_DESCRIPTION", - UpdateOwner = "UPDATE_OWNER", - UpdateTags = "UPDATE_TAGS", -} - -/** - * Advanced configuration for managing owners at multiple hierarchy levels (Service, - * Database, Schema, Table) with custom mappings and inheritance rules. - * - * Configuration for assigning owners to ingested entities following topology hierarchy with - * inheritance support - */ -export interface OwnerConfiguration { - /** - * Owner for database entities. Can be a single owner for all databases, or a map of - * database names to owner(s). - */ - database?: { [key: string]: string[] | string } | string; - /** - * Owner for schema entities. Can be a single owner for all schemas, or a map of schema FQNs - * to owner(s). - */ - databaseSchema?: { [key: string]: string[] | string } | string; - /** - * Default owner applied to all entities when no specific owner is configured (user or team - * name/email) - */ - default?: string; - /** - * Enable child entities to inherit owner from parent entities when they don't have a - * specific owner configured - */ - enableInheritance?: boolean; - /** - * Owner for the service level - */ - service?: string; - /** - * Owner for table entities. Can be a single owner for all tables, or a map of table FQNs to - * owner(s). - */ - table?: { [key: string]: string[] | string } | string; -} - -/** - * Processing Engine Configuration. If not provided, the Native Engine will be used by - * default. - * - * Configuration for the native metadata ingestion engine - * - * This schema defines the configuration for a Spark Engine runner. - */ -export interface ProcessingEngine { - /** - * The type of the engine configuration - */ - type: ProcessingEngineType; - config?: Config; - /** - * Spark Connect Remote URL. - */ - remote?: string; -} - -export interface Config { - /** - * Additional Spark configuration properties as key-value pairs. - */ - extraConfig?: { [key: string]: any }; - /** - * Temporary path to store the data. - */ - tempPath?: string; - [property: string]: any; -} - -/** - * The type of the engine configuration - */ -export enum ProcessingEngineType { - Native = "Native", - Spark = "Spark", -} - -/** - * Type of Profile Sample (percentage or rows) - */ -export enum ProfileSampleType { - Percentage = "PERCENTAGE", - Rows = "ROWS", -} - -/** - * Type of Sampling Method (BERNOULLI or SYSTEM) - */ -export enum SamplingMethodType { - Bernoulli = "BERNOULLI", - System = "SYSTEM", -} - -/** - * Service connections available for the logical test suite. - */ -export interface ServiceConnections { - /** - * Connection configuration for the source. ex: mysql , tableau connection. - */ - serviceConnection: ServiceConnection; - serviceName: string; -} - -/** - * Connection configuration for the source. ex: mysql , tableau connection. - * - * Supported services - * - * API Service Connection. - * - * Dashboard Connection. - * - * Database Connection. - * - * Metadata Service Connection. - * - * Pipeline Connection. - * - * MlModel Connection. - * - * Storage Connection. - * - * search Connection. - * - * Security Connection. - * - * Drive Connection. - */ -export interface ServiceConnection { - config?: ConfigObject; -} - -/** - * REST Connection Config - * - * Looker Connection Config - * - * Metabase Connection Config - * - * PowerBI Connection Config - * - * PowerBIReportServer Connection Config - * - * Redash Connection Config - * - * Superset Connection Config - * - * Tableau Connection Config - * - * Mode Connection Config - * - * Custom Dashboard Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Domo Dashboard Connection Config - * - * QuickSight Connection Config - * - * Qlik Sense Connection Config - * - * Lightdash Connection Config - * - * MicroStrategy Connection Config - * - * Qlik Cloud Connection Config - * - * Sigma Connection Config - * - * ThoughtSpot Connection Config - * - * Grafana Connection Config - * - * Hex Connection Config - * - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * TimescaleDB Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - * - * Kafka Connection Config - * - * Redpanda Connection Config - * - * Kinesis Connection Config - * - * Custom Messaging Service Connection to build a source that is not supported by - * OpenMetadata yet. - * - * Amundsen Connection Config - * - * Metadata to ElasticSearch Connection Config - * - * OpenMetadata Connection Config - * - * Atlas Connection Config - * - * Alation Connection Config - * - * Alation Sink Connection Config - * - * Collibra Connection Config - * - * Airflow Metadata Database Connection Config - * - * Wherescape Metadata Database Connection Config - * - * SSIS Metadata Database Connection Config - * - * Glue Pipeline Connection Config - * - * AWS Kinesis Firehose Pipeline Connection Config - * - * Airbyte Metadata Database Connection Config - * - * Fivetran Metadata Database Connection Config - * - * Flink Metadata Connection Config - * - * Dagster Metadata Database Connection Config - * - * Nifi Metadata Pipeline Connection Config - * - * Domo Pipeline Connection Config - * - * Custom Pipeline Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Spline Metadata Database Connection Config - * - * Spark Metadata Pipeline Connection Config - * - * OpenLineage Connection Config - * - * KafkaConnect Connection Config - * - * DBTCloud Connection Config - * - * Matillion Connection - * - * Azure Data Factory Connection Config - * - * Stitch Connection - * - * Snowplow Pipeline Connection Config - * - * MlFlow Connection Config - * - * Sklearn Connection Config - * - * Custom MlModel Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * SageMaker Connection Config - * - * Google VertexAI Connection Config - * - * S3 Connection. - * - * ADLS Connection. - * - * GCS Connection. - * - * Custom Storage Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * ElasticSearch Connection. - * - * OpenSearch Connection Config - * - * Custom Search Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * Apache Ranger Connection Config - * - * Google Drive Connection Config - * - * SharePoint Connection Config - * - * Custom Drive Connection to build a source that is not supported. - */ -export interface ConfigObject { - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Documentation URL for the schema. - */ - docURL?: string; - /** - * Open API Schema URL. - */ - openAPISchemaURL?: string; - /** - * Supports Metadata Extraction. - */ - supportsMetadataExtraction?: boolean; - /** - * Generated Token to connect to OpenAPI Schema. - * - * token to connect to Qlik Cloud. - * - * Hex API token for authentication. Can be personal or workspace token. - * - * To Connect to Dagster Cloud - * - * Generated Token to connect to Databricks. - * - * Generated Token to connect to DBTCloud. - * - * Token to connect to Stitch api doc - */ - token?: string; - /** - * REST API Type - * - * Service Type - * - * Custom dashboard service type - * - * Custom database service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * OpenSearch Type - * - * Custom search service type - */ - type?: PurpleType; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * User's Client ID. This user should have privileges to read all the metadata in Looker. - * - * client_id for PowerBI. - * - * Client ID for DOMO - * - * client_id for Sigma. - * - * Azure Application (client) ID for service principal authentication. - * - * Application (client) ID from Azure Active Directory - */ - clientId?: string; - /** - * User's Client Secret. - * - * clientSecret for PowerBI. - * - * clientSecret for Sigma. - * - * Azure Application client secret for service principal authentication. - * - * Application (client) secret from Azure Active Directory - */ - clientSecret?: string; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - */ - gitCredentials?: NoGitCredentialsClass | string; - /** - * URL to the Looker instance. - * - * Host and Port of the Metabase instance. - * - * Dashboard URL for PowerBI service. - * - * Dashboard URL for PowerBI Report Server. - * - * URL for the Redash instance - * - * URL for the superset instance. - * - * Tableau Server url. - * - * URL for the mode instance. - * - * URL for the Qlik instance. - * - * Address for your running Lightdash instance - * - * Host and Port of the MicroStrategy instance. - * - * Host and Port of the Qlik Cloud instance. - * - * Sigma API url. - * - * ThoughtSpot instance URL. Example: https://my-company.thoughtspot.cloud - * - * URL to the Grafana instance. - * - * Hex API URL. For Hex.tech cloud, use https://app.hex.tech - * - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the TimescaleDB service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - * - * Host and port of the Amundsen Neo4j Connection. This expect a URI format like: - * bolt://localhost:7687. - * - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - * - * Host and port of the Atlas service. - * - * Host and port of the Alation service. - * - * Host and port of the Collibra service. - * - * Pipeline Service Management/UI URI. - * - * Pipeline Service Management/UI URL. - * - * Spline REST Server Host & Port. - * - * KafkaConnect Service Management/UI URI. - * - * Host and port of the Stitch API host - * - * Host and port of the ElasticSearch service. - * - * Host and port of the OpenSearch service. - * - * Apache Ranger Admin URL. - */ - hostPort?: string; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * API token to connect to Metabase. Use this instead of username/password for token-based - * authentication. - * - * API key of the redash instance to access. - * - * The personal access token you can generate in the Lightdash app under the user settings - * - * Service Account Token to authenticate to the Grafana APIs. Use Service Account Tokens - * (format: glsa_xxxx) for authentication. Legacy API Keys are no longer supported by - * Grafana as of January 2025. Both self-hosted and Grafana Cloud are supported. Requires - * Admin role for full metadata extraction. - * - * API key to authenticate with the SAP ERP APIs. - * - * Fivetran API Key. - * - * API Key for Snowplow Console API - */ - apiKey?: string; - /** - * Password to connect to Metabase. Required for basic authentication. - * - * Password to connect to PowerBI report server. - * - * Password to connect to MicroStrategy. - * - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - * - * password to connect to the Amundsen Neo4j Connection. - * - * password to connect to the Atlas. - * - * Password to connect to the Collibra. - */ - password?: string; - /** - * Username to connect to Metabase. Required for basic authentication. - * - * Username to connect to PowerBI report server. - * - * Username for Redash - * - * Username to connect to MicroStrategy. This user should have privileges to read all the - * metadata in MicroStrategy. - * - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to TimescaleDB. This user should have privileges to read all the - * metadata in TimescaleDB. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to Salesforce. This user should have privileges to read all the - * metadata in Salesforce. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - * - * username to connect to the Amundsen Neo4j Connection. - * - * username to connect to the Atlas. This user should have privileges to read all the - * metadata in Atlas. - * - * Username to connect to the Collibra. This user should have privileges to read all the - * metadata in Collibra. - */ - username?: string; - /** - * API URL to call powerbi rest apis to extract metadata. Default to - * `https://api.powerbi.com`. You can provide youw own in case of different environment - */ - apiURL?: string; - /** - * Authority URI for the PowerBI service. - */ - authorityURI?: string; - /** - * Display Table Name from source instead of renamed table name for datamodel tables - */ - displayTableNameFromSource?: boolean; - /** - * Entity Limit set here will be used to paginate the PowerBi APIs - */ - pagination_entity_per_page?: number; - /** - * Source to get the .pbit files to extract lineage information - */ - pbitFilesSource?: PowerBIPbitFilesSource; - /** - * PowerBI secrets. - */ - scope?: string[]; - /** - * Tenant ID for PowerBI. - * - * Azure Directory (tenant) ID for service principal authentication. - * - * Directory (tenant) ID from Azure Active Directory - */ - tenantId?: string; - /** - * Fetch the PowerBI metadata using admin APIs - */ - useAdminApis?: boolean; - /** - * Web Portal Virtual Directory Name. - */ - webPortalVirtualDirectory?: string; - /** - * Version of the Redash instance - */ - redashVersion?: string; - /** - * Choose between API or database connection fetch metadata from superset. - * - * Choose between Database connection or HDB User Store connection. - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Matillion Auth Configuration - */ - connection?: ConfigConnection; - /** - * Tableau API version. If not provided, the version will be used from the tableau server. - * - * Sigma API version. - * - * ThoughtSpot API version to use - * - * OpenMetadata server API version to use. - * - * Airbyte API version. - */ - apiVersion?: string; - /** - * Types of methods used to authenticate to the tableau instance - * - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - * - * Types of methods used to authenticate to the alation instance - * - * Authentication type to connect to Apache Ranger. - */ - authType?: AuthenticationTypeForTableau | NoConfigAuthenticationTypes; - /** - * Pagination limit used while querying the tableau metadata API for getting data sources - * - * Pagination limit used while querying the SAP ERP API for fetching the entities - * - * Pagination limit used for Alation APIs pagination - */ - paginationLimit?: number; - /** - * Proxy URL for the tableau server. If not provided, the hostPort will be used. This is - * used to generate the dashboard & Chart URL. - */ - proxyURL?: string; - /** - * Tableau Site Name. - * - * SharePoint site name - */ - siteName?: string; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; - /** - * Boolean marking if we need to verify the SSL certs for Grafana. Default to True. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Boolean marking if we need to verify the SSL certs for KafkaConnect REST API. True by - * default. - */ - verifySSL?: boolean | VerifySSL; - /** - * Access Token for Mode Dashboard - * - * Access token to connect to DOMO - */ - accessToken?: string; - /** - * Access Token Password for Mode Dashboard - */ - accessTokenPassword?: string; - /** - * Filter query parameter for some of the Mode API calls - */ - filterQueryParam?: string; - /** - * Mode Workspace Name - */ - workspaceName?: string; - connectionOptions?: { [key: string]: string }; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - * - * Secret token to connect to DOMO - */ - secretToken?: string; - /** - * AWS Account ID - */ - awsAccountId?: string; - awsConfig?: AWSCredentials; - /** - * The authentication method that the user uses to sign in. - */ - identityType?: IdentityType; - /** - * The Amazon QuickSight namespace that contains the dashboard IDs in this request ( To be - * provided when identityType is `ANONYMOUS` ) - */ - namespace?: string; - certificates?: QlikCertificatesBy; - /** - * Qlik Sense Base URL, used for genrating dashboard & chat url - */ - displayUrl?: string; - /** - * User Directory. - */ - userDirectory?: string; - /** - * User ID. - */ - userId?: string; - /** - * Validate Host Name - */ - validateHostName?: boolean; - /** - * The Project UUID for your Lightdash instance - */ - projectUUID?: string; - /** - * Use if your Lightdash instance is behind a proxy like (Cloud IAP) - */ - proxyAuthentication?: string; - /** - * The Space UUID for your Lightdash instance - */ - spaceUUID?: string; - /** - * Login Mode for Microstrategy's REST API connection. You can authenticate with one of the - * following authentication modes: `Standard (1)`, `Anonymous (8)`. Default will be - * `Standard (1)`. If you're using demo account for Microstrategy, it will be needed to - * authenticate through loginMode `8`. - */ - loginMode?: string; - /** - * MicroStrategy Project Name - * - * Project name to create the refreshToken. Can be anything - */ - projectName?: string; - /** - * Space types of Qlik Cloud to filter the dashboards ingested into the platform. - */ - spaceTypes?: SpaceType[]; - /** - * ThoughtSpot authentication configuration - */ - authentication?: Authenticationation; - /** - * Org ID for multi-tenant ThoughtSpot instances. This is applicable for ThoughtSpot Cloud - * only. - */ - orgId?: string; - /** - * Page size for pagination in API requests. Default is 100. - */ - pageSize?: number; - /** - * Whether to import Hex project categories as OpenMetadata tags - * - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Type of token to use for authentication - */ - tokenType?: TokenType; - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - * - * Azure Credentials - * - * GCP Credentials for Google Drive API - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Option to include policy tags as part of column description. - */ - includePolicyTags?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsQueryRunner?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in TimescaleDB. You can use databaseFilterPattern on top - * of this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - * - * Connection timeout in seconds. - */ - connectionTimeout?: number | number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - * - * Available sources to fetch metadata. - */ - configSource?: DeltaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - * - * Choose between Basic authentication (for self-hosted) or OAuth 2.0 client credentials - * (for Airbyte Cloud) - */ - auth?: Authentication | AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - * - * Custom OpenMetadata Classification name for TimescaleDB policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Consumer Key (Client ID) for OAuth 2.0 authentication. This is obtained from - * your Salesforce Connected App configuration. Required along with Consumer Secret for - * OAuth authentication. - */ - consumerKey?: string; - /** - * Salesforce Consumer Secret (Client Secret) for OAuth 2.0 authentication. This is obtained - * from your Salesforce Connected App configuration. Required along with Consumer Key for - * OAuth authentication. - */ - consumerSecret?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - * - * Snowplow BDP Organization ID - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token for username/password authentication. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - /** - * basic.auth.user.info schema registry config property, Client HTTP credentials in the form - * of username:password. - */ - basicAuthUserInfo?: string; - /** - * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - * - * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - */ - bootstrapServers?: string; - /** - * Confluent Kafka Consumer Config. From - * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - * - * Confluent Redpanda Consumer Config - */ - consumerConfig?: { [key: string]: any }; - /** - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - */ - consumerConfigSSL?: ConsumerConfigSSLClass; - /** - * sasl.mechanism Consumer Config property - */ - saslMechanism?: SaslMechanismType; - /** - * sasl.password consumer config property - */ - saslPassword?: string; - /** - * sasl.username consumer config property - */ - saslUsername?: string; - /** - * Confluent Kafka Schema Registry Config. From - * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient - * - * Confluent Redpanda Schema Registry Config. - */ - schemaRegistryConfig?: { [key: string]: any }; - /** - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ - schemaRegistrySSL?: ConsumerConfigSSLClass; - /** - * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get - * topic schema from registry. - */ - schemaRegistryTopicSuffixName?: string; - /** - * Confluent Kafka Schema Registry URL. - * - * Confluent Redpanda Schema Registry URL. - */ - schemaRegistryURL?: string; - /** - * security.protocol consumer config property - * - * Kafka security protocol config - */ - securityProtocol?: KafkaSecurityProtocol; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Enable encryption for the Amundsen Neo4j Connection. - */ - encrypted?: boolean; - /** - * Maximum connection lifetime for the Amundsen Neo4j Connection. - */ - maxConnectionLifeTime?: number; - /** - * Enable SSL validation for the Amundsen Neo4j Connection. - */ - validateSSL?: boolean; - /** - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - /** - * List of entities that you need to reindex - */ - entities?: string[]; - recreateIndex?: boolean; - runMode?: RunMode; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * service type of the data source. - */ - databaseServiceName?: string[]; - /** - * Name of the Entity Type available in Atlas. - */ - entity_type?: string; - /** - * service type of the messaging source - * - * Name of the Kafka Messaging Service associated with this Firehose Pipeline Service. e.g. - * local_kafka - * - * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. - * e.g. local_kafka - */ - messagingServiceName?: string[] | string; - /** - * Custom OpenMetadata Classification name for alation tags. - */ - alationTagClassificationName?: string; - /** - * Specifies if hidden datasources should be included while ingesting. - */ - includeHiddenDatasources?: boolean; - /** - * Specifies if undeployed datasources should be included while ingesting. - */ - includeUndeployedDatasources?: boolean; - /** - * Specifies if Dashboards are to be ingested while running the ingestion job. - */ - ingestDashboards?: boolean; - /** - * Specifies if Datasources are to be ingested while running the ingestion job. - */ - ingestDatasources?: boolean; - /** - * Specifies if Domains are to be ingested while running the ingestion job. - */ - ingestDomains?: boolean; - /** - * Specifies if Knowledge Articles are to be ingested while running the ingestion job. - */ - ingestKnowledgeArticles?: boolean; - /** - * Specifies if Users and Groups are to be ingested while running the ingestion job. - */ - ingestUsersAndGroups?: boolean; - datasourceLinks?: { [key: string]: string }; - /** - * Regex to only include/exclude domains that match the pattern. - */ - domainFilterPattern?: FilterPattern; - /** - * Enable enrichment of existing OpenMetadata assets with Collibra metadata (descriptions, - * tags, owners). When enabled, the connector will match Collibra assets to OpenMetadata - * entities and apply metadata without creating new assets. - */ - enableEnrichment?: boolean; - /** - * Regex to only include/exclude glossaries that match the pattern. - */ - glossaryFilterPattern?: FilterPattern; - /** - * Pipeline Service Number Of Status - */ - numberOfStatus?: number; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Underlying database connection - */ - databaseConnection?: DatabaseConnectionClass; - /** - * Underlying storage connection - */ - packageConnection?: S3Connection | string; - /** - * Fivetran API Secret. - */ - apiSecret?: string; - /** - * Fivetran API Limit For Pagination. - */ - limit?: number; - /** - * URL to the Dagster instance - * - * DBT cloud Access URL. - */ - host?: string; - /** - * Connection Time Limit Between OM and Dagster Graphql API in second - */ - timeout?: number; - /** - * We support username/password or client certificate authentication - */ - nifiConfig?: NifiCredentialsConfiguration; - /** - * Number of days to look back when fetching lineage data from Databricks system tables - * (system.access.table_lineage and system.access.column_lineage). Default is 90 days. - */ - lineageLookBackDays?: number; - /** - * Spline UI Host & Port. - */ - uiHostPort?: string; - /** - * service type of the messaging source - */ - brokersUrl?: string; - /** - * consumer group name - */ - consumerGroupName?: string; - /** - * initial Kafka consumer offset - */ - consumerOffsets?: InitialConsumerOffsets; - /** - * max allowed wait time - */ - poolTimeout?: number; - /** - * SASL Configuration details. - */ - saslConfig?: SASLClientConfig; - /** - * max allowed inactivity time - */ - sessionTimeout?: number; - /** - * topic from where Open lineage events will be pulled - */ - topicName?: string; - /** - * We support username/password or No Authentication - */ - KafkaConnectConfig?: UsernamePasswordAuthentication; - /** - * ID of your DBT cloud account - */ - accountId?: string; - /** - * DBT cloud Metadata API URL. - */ - discoveryAPI?: string; - /** - * List of IDs of your DBT cloud jobs seperated by comma `,` - */ - jobIds?: string[]; - /** - * Number of runs to fetch from DBT cloud - */ - numberOfRuns?: number; - /** - * List of IDs of your DBT cloud projects seperated by comma `,` - */ - projectIds?: string[]; - /** - * The name of your azure data factory. - */ - factory_name?: string; - /** - * The name of your resource group the data factory is associated with. - */ - resource_group_name?: string; - /** - * Number of days in the past to filter pipeline runs. - */ - run_filter_days?: number; - /** - * The azure subscription identifier. - */ - subscription_id?: string; - /** - * Cloud provider where Snowplow is deployed - */ - cloudProvider?: CloudProvider; - /** - * Path to pipeline configuration files for Community deployment - */ - configPath?: string; - /** - * Snowplow Console URL for BDP deployment - */ - consoleUrl?: string; - /** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - */ - deployment?: SnowplowDeployment; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Mlflow Model registry backend. E.g., - * mysql+pymysql://mlflow:password@localhost:3307/experiments - */ - registryUri?: string; - /** - * Mlflow Experiment tracking URI. E.g., http://localhost:5000 - */ - trackingUri?: string; - /** - * location/region of google cloud project - */ - location?: string; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - /** - * Console EndPoint URL for S3-compatible services - */ - consoleEndpointURL?: string; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Connection Timeout in Seconds - */ - connectionTimeoutSecs?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Email to impersonate using domain-wide delegation - */ - delegatedEmail?: string; - /** - * Regex to only include/exclude directories that matches the pattern. - */ - directoryFilterPattern?: FilterPattern; - /** - * Specific shared drive ID to connect to - * - * SharePoint drive ID. If not provided, default document library will be used - */ - driveId?: string; - /** - * Regex to only include/exclude files that matches the pattern. - */ - fileFilterPattern?: FilterPattern; - /** - * Extract metadata only for Google Sheets files - */ - includeGoogleSheets?: boolean; - /** - * Include shared/team drives in metadata extraction - */ - includeTeamDrives?: boolean; - /** - * Regex to only include/exclude spreadsheets that matches the pattern. - */ - spreadsheetFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude worksheets that matches the pattern. - */ - worksheetFilterPattern?: FilterPattern; - /** - * SharePoint site URL - */ - siteUrl?: string; - [property: string]: any; -} - -/** - * We support username/password or No Authentication - * - * username/password auth - */ -export interface UsernamePasswordAuthentication { - /** - * KafkaConnect password to authenticate to the API. - */ - password?: string; - /** - * KafkaConnect user to authenticate to the API. - */ - username?: string; -} - -/** - * Choose between Basic authentication (for self-hosted) or OAuth 2.0 client credentials - * (for Airbyte Cloud) - * - * Username and password authentication - * - * OAuth 2.0 client credentials authentication for Airbyte Cloud - */ -export interface Authentication { - /** - * Password to connect to Airbyte. - */ - password?: string; - /** - * Username to connect to Airbyte. - */ - username?: string; - /** - * Client ID for the application registered in Airbyte. - */ - clientId?: string; - /** - * Client Secret for the application registered in Airbyte. - */ - clientSecret?: string; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Types of methods used to authenticate to the tableau instance - * - * Basic Auth Credentials - * - * Access Token Auth Credentials - * - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - * - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * API Access Token Auth Credentials - * - * Basic Auth Configuration for ElasticSearch - * - * SSL Certificates By Path - * - * AWS credentials configs. - * - * Authentication type to connect to Apache Ranger. - * - * Configuration for connecting to Ranger Basic Auth. - */ -export interface AuthenticationTypeForTableau { - /** - * Password to access the service. - * - * Password to connect to source. - * - * Elastic Search Password for Login - * - * Ranger password to authenticate to the API. - */ - password?: string; - /** - * Username to access the service. - * - * Elastic Search Username for Login - * - * Ranger user to authenticate to the API. - */ - username?: string; - /** - * Personal Access Token Name. - */ - personalAccessTokenName?: string; - /** - * Personal Access Token Secret. - */ - personalAccessTokenSecret?: string; - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; - /** - * Access Token for the API - */ - accessToken?: string; - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -/** - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * Basic Auth Credentials - * - * API Access Token Auth Credentials - */ -export interface Authenticationation { - /** - * Password to access the service. - */ - password?: string; - /** - * Username to access the service. - */ - username?: string; - /** - * Access Token for the API - */ - accessToken?: string; -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: AuthenticationEnum; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum AuthenticationEnum { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: AWSCredentialsClass | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AWSCredentialsClass { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Qlik Authentication Certificate By Values - * - * Qlik Authentication Certificate File Path - */ -export interface QlikCertificatesBy { - sslConfig?: ConsumerConfigSSLClass; - /** - * Client Certificate - */ - clientCertificate?: string; - /** - * Client Key Certificate. - */ - clientKeyCertificate?: string; - /** - * Root Certificate. - */ - rootCertificate?: string; - [property: string]: any; -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface ConsumerConfigSSLClass { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Cloud provider where Snowplow is deployed - */ -export enum CloudProvider { - Aws = "AWS", - Azure = "Azure", - Gcp = "GCP", -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface DeltaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConfigSourceConnection; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: Credentials; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConfigSourceConnection { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Choose between API or database connection fetch metadata from superset. - * - * Superset API Connection Config - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - * - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Lineage Backend Connection Config - * - * SQLite Database Connection Config - * - * Matillion Auth Configuration - * - * Matillion ETL Auth Config. - */ -export interface ConfigConnection { - /** - * Password for Superset. - * - * Password to connect to Hana. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to the Matillion. - */ - password?: string; - /** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ - provider?: Provider; - /** - * SSL Configuration details. - */ - sslConfig?: ConnectionSSLConfig; - /** - * Username for Superset. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to Hana. This user should have privileges to read all the metadata. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to the Matillion. This user should have privileges to read all the - * metadata in Matillion. - */ - username?: string; - verifySSL?: VerifySSL; - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - * - * Host and port of the Hana service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Matillion Host - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: ConnectionScheme; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: ConnectionType; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - supportsViewLineageExtraction?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ -export enum Provider { - DB = "db", - LDAP = "ldap", -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum ConnectionScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", - SqlitePysqlite = "sqlite+pysqlite", -} - -/** - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - */ -export interface ConnectionSSLConfig { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum ConnectionType { - Backend = "Backend", - MatillionETL = "MatillionETL", - Mysql = "Mysql", - Postgres = "Postgres", - SQLite = "SQLite", -} - -/** - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * initial Kafka consumer offset - */ -export enum InitialConsumerOffsets { - Earliest = "earliest", - Latest = "latest", -} - -/** - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Underlying database connection - * - * Mssql Database Connection Config - */ -export interface DatabaseConnectionClass { - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Host and port of the MSSQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to MSSQL. - */ - password?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: MssqlScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: MssqlType; - /** - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - */ - username?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum MssqlScheme { - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", -} - -/** - * Service Type - * - * Service type. - */ -export enum MssqlType { - Mssql = "Mssql", -} - -/** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - * - * Snowplow deployment type - */ -export enum SnowplowDeployment { - Bdp = "BDP", - Community = "Community", -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Do not set any credentials. Note that credentials are required to extract .lkml views and - * their lineage. - * - * Credentials for a GitHub repository - * - * Credentials for a BitBucket repository - * - * Credentials for a Gitlab repository - */ -export interface NoGitCredentialsClass { - /** - * GitHub instance URL. For GitHub.com, use https://github.com - * - * BitBucket instance URL. For BitBucket Cloud, use https://bitbucket.org - * - * Gitlab instance URL. For Gitlab.com, use https://gitlab.com - */ - gitHostURL?: string; - repositoryName?: string; - repositoryOwner?: string; - token?: string; - /** - * Credentials Type - */ - type?: NoGitCredentialsType; - /** - * Main production branch of the repository. E.g., `main` - */ - branch?: string; -} - -/** - * Credentials Type - * - * GitHub Credentials type - * - * BitBucket Credentials type - * - * Gitlab Credentials type - */ -export enum NoGitCredentialsType { - BitBucket = "BitBucket", - GitHub = "GitHub", - Gitlab = "Gitlab", -} - -/** - * The authentication method that the user uses to sign in. - */ -export enum IdentityType { - Anonymous = "ANONYMOUS", - Iam = "IAM", - Quicksight = "QUICKSIGHT", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: ConsumerConfigSSLClass; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * We support username/password or client certificate authentication - * - * Configuration for connecting to Nifi Basic Auth. - * - * Configuration for connecting to Nifi Client Certificate Auth. - */ -export interface NifiCredentialsConfiguration { - /** - * Nifi password to authenticate to the API. - */ - password?: string; - /** - * Nifi user to authenticate to the API. - */ - username?: string; - /** - * Boolean marking if we need to verify the SSL certs for Nifi. False by default. - */ - verifySSL?: boolean; - /** - * Path to the root CA certificate - */ - certificateAuthorityPath?: string; - /** - * Path to the client certificate - */ - clientCertificatePath?: string; - /** - * Path to the client key - */ - clientkeyPath?: string; -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * S3 Connection. - */ -export interface S3Connection { - awsConfig: AWSCredentials; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Console EndPoint URL for S3-compatible services - */ - consoleEndpointURL?: string; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: S3Type; -} - -/** - * Service Type - * - * S3 service type - */ -export enum S3Type { - S3 = "S3", -} - -/** - * Source to get the .pbit files to extract lineage information - * - * Local config source where no extra information needs to be sent. - * - * Azure storage config for pbit files - * - * GCS storage config for pbit files - * - * S3 storage config for pbit files - */ -export interface PowerBIPbitFilesSource { - /** - * Directory path for the pbit files - */ - path?: string; - /** - * pbit File Configuration type - */ - pbitFileConfigType?: PbitFileConfigType; - /** - * Path of the folder where the .pbit files will be unzipped and datamodel schema will be - * extracted - */ - pbitFilesExtractDir?: string; - prefixConfig?: BucketDetails; - securityConfig?: Credentials; -} - -/** - * pbit File Configuration type - */ -export enum PbitFileConfigType { - Azure = "azure", - Gcs = "gcs", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the .pbit files are stored - */ -export interface BucketDetails { - /** - * Name of the bucket where the .pbit files are stored - */ - bucketName?: string; - /** - * Path of the folder where the .pbit files are stored - */ - objectPrefix?: string; -} - -/** - * This schema publisher run modes. - */ -export enum RunMode { - Batch = "batch", - Stream = "stream", -} - -/** - * SASL Configuration details. - * - * SASL client configuration. - */ -export interface SASLClientConfig { - /** - * SASL security mechanism - */ - saslMechanism?: SaslMechanismType; - /** - * The SASL authentication password. - */ - saslPassword?: string; - /** - * The SASL authentication username. - */ - saslUsername?: string; -} - -/** - * sasl.mechanism Consumer Config property - * - * SASL Mechanism consumer config property - * - * SASL security mechanism - */ -export enum SaslMechanismType { - Gssapi = "GSSAPI", - Oauthbearer = "OAUTHBEARER", - Plain = "PLAIN", - ScramSHA256 = "SCRAM-SHA-256", - ScramSHA512 = "SCRAM-SHA-512", -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * security.protocol consumer config property - * - * Kafka security protocol config - */ -export enum KafkaSecurityProtocol { - Plaintext = "PLAINTEXT", - SSL = "SSL", - SaslPlaintext = "SASL_PLAINTEXT", - SaslSSL = "SASL_SSL", -} - -export enum SpaceType { - Data = "Data", - Managed = "Managed", - Personal = "Personal", - Shared = "Shared", -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Config - */ -export interface SSLConfigObject { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; - /** - * SSL Certificates - */ - certificates?: SSLCertificates; - [property: string]: any; -} - -/** - * SSL Certificates - * - * SSL Configuration details. - * - * SSL Certificates By Path - * - * SSL Certificates By Values - */ -export interface SSLCertificates { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * CA Certificate Value - */ - caCertValue?: string; - /** - * Client Certificate Value - */ - clientCertValue?: string; - /** - * Private Key Value - */ - privateKeyValue?: string; - /** - * Staging Directory Path - */ - stagingDir?: string; -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * Type of token to use for authentication - */ -export enum TokenType { - Personal = "personal", - Workspace = "workspace", -} - -/** - * REST API Type - * - * REST API type - * - * Service Type - * - * Looker service type - * - * Metabase service type - * - * PowerBI service type - * - * PowerBIReportServer service type - * - * Redash service type - * - * Superset service type - * - * Tableau service type - * - * Mode service type - * - * Custom dashboard service type - * - * service type - * - * QuickSight service type - * - * Qlik sense service type - * - * Lightdash service type - * - * MicroStrategy service type - * - * Qlik Cloud service type - * - * Sigma service type - * - * ThoughtSpot service type - * - * Grafana service type - * - * Service type. - * - * Custom database service type - * - * Kafka service type - * - * Redpanda service type - * - * Custom messaging service type - * - * Amundsen service type - * - * Metadata to Elastic Search type - * - * OpenMetadata service type - * - * Collibra service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * S3 service type - * - * ADLS service type - * - * Gcs service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * ElasticSearch service type - * - * OpenSearch Type - * - * OpenSearch service type - * - * Custom search service type - * - * Apache Ranger service type - * - * Google Drive service type - * - * SharePoint service type - * - * Custom Drive service type - */ -export enum PurpleType { - Adls = "ADLS", - Airbyte = "Airbyte", - Airflow = "Airflow", - Alation = "Alation", - AlationSink = "AlationSink", - Amundsen = "Amundsen", - Athena = "Athena", - Atlas = "Atlas", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Collibra = "Collibra", - Couchbase = "Couchbase", - CustomDashboard = "CustomDashboard", - CustomDatabase = "CustomDatabase", - CustomDrive = "CustomDrive", - CustomMessaging = "CustomMessaging", - CustomMlModel = "CustomMlModel", - CustomPipeline = "CustomPipeline", - CustomSearch = "CustomSearch", - CustomStorage = "CustomStorage", - DBTCloud = "DBTCloud", - Dagster = "Dagster", - DataFactory = "DataFactory", - Databricks = "Databricks", - DatabricksPipeline = "DatabricksPipeline", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDashboard = "DomoDashboard", - DomoDatabase = "DomoDatabase", - DomoPipeline = "DomoPipeline", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - ElasticSearch = "ElasticSearch", - Epic = "Epic", - Exasol = "Exasol", - Fivetran = "Fivetran", - Flink = "Flink", - Gcs = "GCS", - Glue = "Glue", - GluePipeline = "GluePipeline", - GoogleDrive = "GoogleDrive", - Grafana = "Grafana", - Greenplum = "Greenplum", - Hex = "Hex", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - Kafka = "Kafka", - KafkaConnect = "KafkaConnect", - Kinesis = "Kinesis", - KinesisFirehose = "KinesisFirehose", - Lightdash = "Lightdash", - Looker = "Looker", - MariaDB = "MariaDB", - Matillion = "Matillion", - Metabase = "Metabase", - MetadataES = "MetadataES", - MicroStrategy = "MicroStrategy", - Mlflow = "Mlflow", - Mode = "Mode", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Nifi = "Nifi", - OpenLineage = "OpenLineage", - OpenMetadata = "OpenMetadata", - OpenSearch = "OpenSearch", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - PowerBI = "PowerBI", - PowerBIReportServer = "PowerBIReportServer", - Presto = "Presto", - QlikCloud = "QlikCloud", - QlikSense = "QlikSense", - QuickSight = "QuickSight", - REST = "Rest", - Ranger = "Ranger", - Redash = "Redash", - Redpanda = "Redpanda", - Redshift = "Redshift", - S3 = "S3", - SAS = "SAS", - SQLite = "SQLite", - SageMaker = "SageMaker", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SharePoint = "SharePoint", - Sigma = "Sigma", - SingleStore = "SingleStore", - Sklearn = "Sklearn", - Snowflake = "Snowflake", - Snowplow = "Snowplow", - Spark = "Spark", - Spline = "Spline", - Ssas = "SSAS", - Ssis = "SSIS", - Stitch = "Stitch", - Superset = "Superset", - Synapse = "Synapse", - Tableau = "Tableau", - Teradata = "Teradata", - ThoughtSpot = "ThoughtSpot", - Timescale = "Timescale", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - VertexAI = "VertexAI", - Vertica = "Vertica", - Wherescape = "Wherescape", -} - -/** - * No manifest file available. Ingestion would look for bucket-level metadata file instead - * - * Storage Metadata Manifest file path config. - * - * Storage Metadata Manifest file HTTP path config. - * - * Storage Metadata Manifest file S3 path config. - * - * Storage Metadata Manifest file ADLS path config. - * - * Storage Metadata Manifest file GCS path config. - */ -export interface StorageMetadataConfigurationSource { - /** - * Storage Metadata manifest file path to extract locations to ingest from. - */ - manifestFilePath?: string; - /** - * Storage Metadata manifest http file path to extract locations to ingest from. - */ - manifestHttpPath?: string; - prefixConfig?: StorageMetadataBucketDetails; - securityConfig?: Credentials; -} - -/** - * Details of the bucket where the storage metadata manifest file is stored - */ -export interface StorageMetadataBucketDetails { - /** - * Name of the top level container where the storage metadata file is stored - */ - containerName: string; - /** - * Path of the folder where the storage metadata file is stored. If the file is at the root, - * you can keep it empty. - */ - objectPrefix?: string; -} - -/** - * Pipeline type - * - * Database Source Config Metadata Pipeline type - * - * Database Source Config Usage Pipeline type - * - * Dashboard Source Config Metadata Pipeline type - * - * Messaging Source Config Metadata Pipeline type - * - * Profiler Source Config Pipeline type - * - * Pipeline Source Config Metadata Pipeline type - * - * MlModel Source Config Metadata Pipeline type - * - * Object Store Source Config Metadata Pipeline type - * - * Drive Source Config Metadata Pipeline type - * - * Search Source Config Metadata Pipeline type - * - * DBT Config Pipeline type - * - * Pipeline Source Config For Application Pipeline type. Nothing is required. - * - * Api Source Config Metadata Pipeline type - * - * Reverse Ingestion Config Pipeline type - */ -export enum FluffyType { - APIMetadata = "ApiMetadata", - Application = "Application", - AutoClassification = "AutoClassification", - DashboardMetadata = "DashboardMetadata", - DataInsight = "dataInsight", - DatabaseLineage = "DatabaseLineage", - DatabaseMetadata = "DatabaseMetadata", - DatabaseUsage = "DatabaseUsage", - Dbt = "DBT", - DriveMetadata = "DriveMetadata", - MessagingMetadata = "MessagingMetadata", - MetadataToElasticSearch = "MetadataToElasticSearch", - MlModelMetadata = "MlModelMetadata", - PipelineMetadata = "PipelineMetadata", - Profiler = "Profiler", - ReverseIngestion = "ReverseIngestion", - SearchMetadata = "SearchMetadata", - StorageMetadata = "StorageMetadata", - TestSuite = "TestSuite", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/slackAppConfiguration.ts b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/slackAppConfiguration.ts index 9dba21ea5d8..8f43173d665 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/slackAppConfiguration.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/slackAppConfiguration.ts @@ -14,6 +14,10 @@ * This schema defines the Slack App Information */ export interface SlackAppConfiguration { + /** + * Bot Token + */ + botToken?: string; /** * Client Id of the Application */ @@ -27,4 +31,8 @@ export interface SlackAppConfiguration { * verifying its unique signature. */ signingSecret: string; + /** + * User Token + */ + userToken?: string; } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts deleted file mode 100644 index ac1de504543..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts +++ /dev/null @@ -1,1598 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This schema defines the applications for Open-Metadata. - */ -export interface App { - /** - * This schema defines the type of the agent. - */ - agentType?: AgentType; - /** - * If true, multiple instances of this app can run concurrently. This is useful for apps - * like QueryRunner that support parallel executions with different configurations. - */ - allowConcurrentExecution?: boolean; - /** - * Allow users to configure the app from the UI. If `false`, the `configure` step will be - * hidden. - */ - allowConfiguration?: boolean; - /** - * Application Configuration object. - */ - appConfiguration?: any[] | boolean | number | null | CollateAIAppConfig | string; - /** - * Application Logo Url. - */ - appLogoUrl?: string; - /** - * In case the app supports scheduling, list of different app schedules - */ - appSchedule?: any[] | boolean | AppScheduleClass | number | number | null | string; - /** - * Application Screenshots. - */ - appScreenshots?: string[]; - /** - * This schema defines the type of application. - */ - appType: AppType; - /** - * Bot User Associated with this application. - */ - bot?: EntityReference; - /** - * Change that lead to this version of the entity. - */ - changeDescription?: ChangeDescription; - /** - * Fully Qualified ClassName for the Schedule - */ - className: string; - /** - * When `true` indicates the entity has been soft deleted. - */ - deleted?: boolean; - /** - * Description of the Application. - */ - description?: string; - /** - * Developer For the Application. - */ - developer?: string; - /** - * Url for the developer - */ - developerUrl?: string; - /** - * Display Name for the application. - */ - displayName?: string; - /** - * Domains the asset belongs to. When not set, the asset inherits the domain from the parent - * it belongs to. - */ - domains?: EntityReference[]; - /** - * Event Subscriptions for the Application. - */ - eventSubscriptions?: EntityReference[]; - /** - * Features of the Application. - */ - features?: string; - /** - * FullyQualifiedName same as `name`. - */ - fullyQualifiedName?: string; - /** - * Link to the resource corresponding to this entity. - */ - href?: string; - /** - * Unique identifier of this application. - */ - id: string; - /** - * Bot user that performed the action on behalf of the actual user. - */ - impersonatedBy?: string; - /** - * Change that lead to this version of the entity. - */ - incrementalChangeDescription?: ChangeDescription; - /** - * The ingestion agent responsible for executing the ingestion pipeline. It will be defined - * at runtime based on the Ingestion Agent of the service. - */ - ingestionRunner?: EntityReference; - /** - * Name of the Application. - */ - name: string; - openMetadataServerConnection?: OpenMetadataConnection; - /** - * Owners of this workflow. - */ - owners?: EntityReference[]; - /** - * Permission used by Native Applications. - */ - permission: Permissions; - /** - * References to pipelines deployed for this database service to extract metadata, usage, - * lineage etc.. - */ - pipelines?: EntityReference[]; - /** - * Flag to enable/disable preview for the application. If the app is in preview mode, it - * can't be installed. - */ - preview?: boolean; - /** - * Privacy Policy for the developer - */ - privacyPolicyUrl?: string; - /** - * Application Private configuration loaded at runtime. - */ - privateConfiguration?: PrivateConfig; - provider?: ProviderType; - /** - * Execution Configuration. - */ - runtime: ExecutionContext; - /** - * This schema defines the Schedule Type of Application. - */ - scheduleType: ScheduleType; - /** - * Fully Qualified class name for the Python source that will execute the external - * application. - */ - sourcePythonClass?: string; - /** - * Support Email for the application - */ - supportEmail?: string; - /** - * If the app support execution through the external runner. - */ - supportsIngestionRunner?: boolean; - /** - * If the app run can be interrupted as part of the execution. - */ - supportsInterrupt?: boolean; - /** - * A system app cannot be uninstalled or modified. - */ - system?: boolean; - /** - * Last update time corresponding to the new version of the entity in Unix epoch time - * milliseconds. - */ - updatedAt?: number; - /** - * User who made the update. - */ - updatedBy?: string; - /** - * Metadata version of the entity. - */ - version?: number; -} - -/** - * This schema defines the type of the agent. - * - * This schema defines the type of application. - */ -export enum AgentType { - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - CollateAITierAgent = "CollateAITierAgent", - Metadata = "Metadata", -} - -/** - * Configuration for the CollateAI External Application. - * - * Configuration for the Automator External Application. - * - * This schema defines the Slack App Token Configuration - * - * Configuration for the Collate AI Quality Agent. - * - * No configuration needed to instantiate the Data Insights Pipeline. The logic is handled - * in the backend. - * - * Search Indexing App. - * - * Cache Warmup Application Configuration. - * - * Configuration for the AutoPilot Application. - */ -export interface CollateAIAppConfig { - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - filter?: string; - /** - * Patch the description if it is empty, instead of raising a suggestion - * - * Patch the tier if it is empty, instead of raising a suggestion - */ - patchIfEmpty?: boolean; - /** - * Application Type - */ - type?: Type; - /** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - */ - actions?: Action[]; - /** - * Entities selected to run the automation. - */ - resources?: Resource; - /** - * Bot Token - */ - botToken?: string; - /** - * User Token - */ - userToken?: string; - /** - * Whether the suggested tests should be active or not upon suggestion - * - * Whether the AutoPilot Workflow should be active or not. - */ - active?: boolean; - backfillConfiguration?: BackfillConfiguration; - /** - * Maximum number of events processed at a time (Default 100). - * - * Maximum number of events sent in a batch (Default 100). - * - * Number of entities to process in each batch. - */ - batchSize?: number; - moduleConfiguration?: ModuleConfiguration; - /** - * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property - * Type and are facing errors. Bear in mind that recreating the index will delete your - * DataAssets and a backfill will be needed. - */ - recreateDataAssetsIndex?: boolean; - sendToAdmins?: boolean; - sendToTeams?: boolean; - /** - * Enable automatic performance tuning based on cluster capabilities and database entity - * count - */ - autoTune?: boolean; - /** - * Number of threads to use for reindexing - * - * Number of parallel threads for processing entities and warming cache. - */ - consumerThreads?: number; - /** - * List of Entities to Reindex - * - * List of entity types to warm up in cache. Use 'all' to warm up all entity types. - */ - entities?: string[]; - /** - * Initial backoff time in milliseconds - */ - initialBackoff?: number; - /** - * Maximum backoff time in milliseconds - */ - maxBackoff?: number; - /** - * Maximum number of concurrent requests to the search index - */ - maxConcurrentRequests?: number; - /** - * Maximum number of retries for a failed request - */ - maxRetries?: number; - /** - * Maximum number of events sent in a batch (Default 100). - */ - payLoadSize?: number; - /** - * Number of threads to use for reindexing - */ - producerThreads?: number; - /** - * Queue Size to user internally for reindexing. - * - * Internal queue size for entity processing pipeline. - */ - queueSize?: number; - /** - * This schema publisher run modes. - */ - recreateIndex?: boolean; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Force cache warmup even if another instance is detected (use with caution). - */ - force?: boolean; - /** - * Enter the retention period for Activity Threads of type = 'Conversation' records in days - * (e.g., 30 for one month, 60 for two months). - */ - activityThreadsRetentionPeriod?: number; - /** - * Enter the retention period for change event records in days (e.g., 7 for one week, 30 for - * one month). - */ - changeEventRetentionPeriod?: number; - /** - * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two - * months). - */ - profileDataRetentionPeriod?: number; - /** - * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for - * two months). - */ - testCaseResultsRetentionPeriod?: number; - /** - * Service Entity Link for which to trigger the application. - */ - entityLink?: string; - [property: string]: any; -} - -/** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - * - * Apply Classification Tags to the selected assets. - * - * Remove Classification Tags Action Type - * - * Apply Glossary Terms to the selected assets. - * - * Remove Glossary Terms Action Type - * - * Add domains to the selected assets. - * - * Remove domains from the selected assets. - * - * Apply Tags to the selected assets. - * - * Add a Custom Property to the selected assets. - * - * Remove Owner Action Type - * - * Add an owner to the selected assets. - * - * Add Test Cases to the selected assets. - * - * Remove Test Cases Action Type - * - * Add owners to the selected assets. - * - * Remove Custom Properties Action Type - * - * Add a Data Product to the selected assets. - * - * Remove a Data Product to the selected assets. - * - * Propagate description, tags and glossary terms via lineage - * - * ML Tagging action configuration for external automator. - */ -export interface Action { - /** - * Apply tags to the children of the selected assets that match the criteria. E.g., columns, - * tasks, topic fields,... - * - * Remove tags from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply terms to the children of the selected assets that match the criteria. E.g., - * columns, tasks, topic fields,... - * - * Remove terms from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply the description to the children of the selected assets that match the criteria. - * E.g., columns, tasks, topic fields,... - * - * Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Add tests to the selected table columns - * - * Remove tests to the selected table columns - */ - applyToChildren?: string[]; - /** - * Update tags even if they are already defined in the asset. By default, incoming tags are - * merged with the existing ones. - * - * Update terms even if they are already defined in the asset. By default, incoming terms - * are merged with the existing ones. - * - * Update the domains even if they are defined in the asset. By default, we will only apply - * the domains to assets without domains. - * - * Update the description even if they are already defined in the asset. By default, we'll - * only add the descriptions to assets without the description set. - * - * Update the Custom Property even if it is defined in the asset. By default, we will only - * apply the owners to assets without the given Custom Property informed. - * - * Update the tier even if it is defined in the asset. By default, we will only apply the - * tier to assets without tier. - * - * Update the test even if it is defined in the asset. By default, we will only apply the - * test to assets without the existing test already existing. - * - * Update the owners even if it is defined in the asset. By default, we will only apply the - * owners to assets without owner. - * - * Update the Data Product even if the asset belongs to a different Domain. By default, we - * will only add the Data Product if the asset has no Domain, or it belongs to the same - * domain as the Data Product. - * - * Update descriptions, tags and Glossary Terms via lineage even if they are already defined - * in the asset. By default, descriptions are only updated if they are not already defined - * in the asset, and incoming tags are merged with the existing ones. - */ - overwriteMetadata?: boolean; - /** - * Classification Tags to apply (source must be 'Classification') - * - * Classification Tags to remove (source must be 'Classification') - */ - tags?: TierElement[]; - /** - * Application Type - */ - type: ActionType; - /** - * Remove tags from all the children and parent of the selected assets. - * - * Remove terms from all the children and parent of the selected assets. - * - * Remove descriptions from all the children and parent of the selected assets. - */ - applyToAll?: boolean; - /** - * Remove tags by its label type - * - * Remove terms by its label type - */ - labels?: LabelElement[]; - /** - * Glossary Terms to apply - * - * Glossary Terms to remove - */ - terms?: TierElement[]; - /** - * Domains to apply - */ - domains?: EntityReference[]; - /** - * Description to apply - */ - description?: string; - /** - * Owners to apply - * - * Custom Properties keys to remove - */ - customProperties?: any; - /** - * tier to apply - */ - tier?: TierElement; - /** - * Test Cases to apply - */ - testCases?: TestCaseDefinitions[]; - /** - * Remove all test cases - */ - removeAll?: boolean; - /** - * Test Cases to remove - */ - testCaseDefinitions?: string[]; - /** - * Owners to apply - */ - owners?: EntityReference[]; - /** - * Data Products to apply - * - * Data Products to remove - */ - dataProducts?: EntityReference[]; - /** - * Propagate the metadata to columns via column-level lineage. - */ - propagateColumnLevel?: boolean; - /** - * Propagate description through lineage - */ - propagateDescription?: boolean; - /** - * Propagate domains from the parent through lineage - */ - propagateDomains?: boolean; - /** - * Propagate glossary terms through lineage - */ - propagateGlossaryTerms?: boolean; - /** - * Propagate owner from the parent - */ - propagateOwner?: boolean; - /** - * Propagate the metadata to the parents (e.g., tables) via lineage. - */ - propagateParent?: boolean; - /** - * Propagate tags through lineage - */ - propagateTags?: boolean; - /** - * Propagate tier from the parent - */ - propagateTier?: boolean; - /** - * Number of levels to propagate lineage. If not set, it will propagate to all levels. - */ - propagationDepth?: number; - /** - * List of configurations to stop propagation based on conditions - */ - propagationStopConfigs?: PropagationStopConfig[]; -} - -/** - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Bot User Associated with this application. - * - * The ingestion agent responsible for executing the ingestion pipeline. It will be defined - * at runtime based on the Ingestion Agent of the service. - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Remove tags by its label type - * - * Remove terms by its label type - */ -export enum LabelElement { - Automated = "Automated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Configuration to stop lineage propagation based on conditions - */ -export interface PropagationStopConfig { - /** - * The metadata attribute to check for stopping propagation - */ - metadataAttribute: MetadataAttribute; - /** - * List of attribute values that will stop propagation when any of them is matched - */ - value: Array; -} - -/** - * The metadata attribute to check for stopping propagation - */ -export enum MetadataAttribute { - Description = "description", - Domains = "domains", - GlossaryTerms = "glossaryTerms", - Owner = "owner", - Tags = "tags", - Tier = "tier", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - * - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Bot User Associated with this application. - * - * The ingestion agent responsible for executing the ingestion pipeline. It will be defined - * at runtime based on the Ingestion Agent of the service. - */ -export interface TagLabel { - /** - * Description for the tag label. - * - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this tag. - * - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Link to the tag resource. - * - * Link to the entity resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType?: LabelTypeEnum; - /** - * Name of the tag or glossary term. - * - * Name of the entity instance. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source?: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state?: State; - style?: Style; - tagFQN?: string; - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id?: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type?: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelTypeEnum { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * Cover image configuration for the entity. - */ - coverImage?: CoverImage; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * Cover image configuration for the entity. - * - * Cover image configuration for an entity. This is used to display a banner or header image - * for entities like Domain, Glossary, Data Product, etc. - */ -export interface CoverImage { - /** - * Position of the cover image in CSS background-position format. Supports keywords (top, - * center, bottom) or pixel values (e.g., '20px 30px'). - */ - position?: string; - /** - * URL of the cover image. - */ - url?: string; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - */ -export interface TierElement { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelTypeEnum; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Minimum set of requirements to get a Test Case request ready - */ -export interface TestCaseDefinitions { - /** - * Compute the passed and failed row count for the test case. - */ - computePassedFailedRowCount?: boolean; - parameterValues?: TestCaseParameterValue[]; - /** - * Tags to apply - */ - tags?: TierElement[]; - /** - * Fully qualified name of the test definition. - */ - testDefinition?: string; - /** - * If the test definition supports it, use dynamic assertion to evaluate the test case. - */ - useDynamicAssertion?: boolean; - [property: string]: any; -} - -/** - * This schema defines the parameter values that can be passed for a Test Case. - */ -export interface TestCaseParameterValue { - /** - * name of the parameter. Must match the parameter names in testCaseParameterDefinition - */ - name?: string; - /** - * value to be passed for the Parameters. These are input from Users. We capture this in - * string and convert during the runtime. - */ - value?: string; - [property: string]: any; -} - -/** - * Application Type - * - * Add Tags action type. - * - * Remove Classification Tags Action Type. - * - * Add Terms action type. - * - * Remove Terms Action Type. - * - * Add Domain Action Type. - * - * Remove Domain Action Type - * - * Add Description Action Type. - * - * Add Custom Properties Action Type. - * - * Remove Description Action Type - * - * Add Tier Action Type. - * - * Remove Tier Action Type - * - * Add Test Case Action Type. - * - * Remove Test Case Action Type - * - * Add Owner Action Type. - * - * Remove Owner Action Type - * - * Remove Custom Properties Action Type. - * - * Add Data Products Action Type. - * - * Remove Data Products Action Type. - * - * Lineage propagation action type. - * - * ML PII Tagging action type. - */ -export enum ActionType { - AddCustomPropertiesAction = "AddCustomPropertiesAction", - AddDataProductAction = "AddDataProductAction", - AddDescriptionAction = "AddDescriptionAction", - AddDomainAction = "AddDomainAction", - AddOwnerAction = "AddOwnerAction", - AddTagsAction = "AddTagsAction", - AddTermsAction = "AddTermsAction", - AddTestCaseAction = "AddTestCaseAction", - AddTierAction = "AddTierAction", - LineagePropagationAction = "LineagePropagationAction", - MLTaggingAction = "MLTaggingAction", - RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", - RemoveDataProductAction = "RemoveDataProductAction", - RemoveDescriptionAction = "RemoveDescriptionAction", - RemoveDomainAction = "RemoveDomainAction", - RemoveOwnerAction = "RemoveOwnerAction", - RemoveTagsAction = "RemoveTagsAction", - RemoveTermsAction = "RemoveTermsAction", - RemoveTestCaseAction = "RemoveTestCaseAction", - RemoveTierAction = "RemoveTierAction", -} - -/** - * Backfill Configuration - */ -export interface BackfillConfiguration { - /** - * Enable Backfill for the configured dates - */ - enabled?: boolean; - /** - * Date for which the backfill will end - */ - endDate?: Date; - /** - * Date from which to start the backfill - */ - startDate?: Date; - [property: string]: any; -} - -/** - * Different Module Configurations - */ -export interface ModuleConfiguration { - /** - * App Analytics Module configuration - */ - appAnalytics: AppAnalyticsConfig; - /** - * Cost Analysis Insights Module configuration - */ - costAnalysis: CostAnalysisConfig; - /** - * Data Assets Insights Module configuration - */ - dataAssets: DataAssetsConfig; - /** - * Data Quality Insights Module configuration - */ - dataQuality: DataQualityConfig; -} - -/** - * App Analytics Module configuration - */ -export interface AppAnalyticsConfig { - /** - * If Enabled, App Analytics insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Cost Analysis Insights Module configuration - */ -export interface CostAnalysisConfig { - /** - * If Enabled, Cost Analysis insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Data Assets Insights Module configuration - */ -export interface DataAssetsConfig { - /** - * If Enabled, Data Asset insights will be populated when the App runs. - */ - enabled: boolean; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Defines the number of days the Data Assets Insights information will be kept. After it - * they will be deleted. - */ - retention?: number; - serviceFilter?: ServiceFilter; -} - -export interface ServiceFilter { - serviceName?: string; - serviceType?: string; -} - -/** - * Data Quality Insights Module configuration - */ -export interface DataQualityConfig { - /** - * If Enabled, Data Quality insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Entities selected to run the automation. - */ -export interface Resource { - /** - * Filter JSON tree to be used for rendering the filters in the UI. This comes from - * Immutable Tree type of react-awesome-query-builder. - */ - filterJsonTree?: string; - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - queryFilter?: string; - /** - * Type of the entity. E.g., 'table', 'chart',... - */ - type?: string[]; - [property: string]: any; -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Application Type - * - * Application type. - */ -export enum Type { - AutoPilotApplication = "AutoPilotApplication", - Automator = "Automator", - CacheWarmup = "CacheWarmup", - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - CollateAITierAgent = "CollateAITierAgent", - DataInsights = "DataInsights", - DataInsightsReport = "DataInsightsReport", - SearchIndexing = "SearchIndexing", -} - -export interface AppScheduleClass { - /** - * Cron Expression in case of Custom scheduled Trigger - */ - cronExpression?: string; - scheduleTimeline: ScheduleTimeline; -} - -/** - * This schema defines the Application ScheduleTimeline Options - */ -export enum ScheduleTimeline { - Custom = "Custom", - Daily = "Daily", - Hourly = "Hourly", - Monthly = "Monthly", - None = "None", - Weekly = "Weekly", -} - -/** - * This schema defines the type of application. - */ -export enum AppType { - External = "external", - Internal = "internal", -} - -/** - * Change that lead to this version of the entity. - * - * Description of the change. - */ -export interface ChangeDescription { - changeSummary?: { [key: string]: ChangeSummary }; - /** - * Names of fields added during the version changes. - */ - fieldsAdded?: FieldChange[]; - /** - * Fields deleted during the version changes with old value before deleted. - */ - fieldsDeleted?: FieldChange[]; - /** - * Fields modified during the version changes with old and new values. - */ - fieldsUpdated?: FieldChange[]; - /** - * When a change did not result in change, this could be same as the current version. - */ - previousVersion?: number; -} - -export interface ChangeSummary { - changedAt?: number; - /** - * Name of the user or bot who made this change - */ - changedBy?: string; - changeSource?: ChangeSource; - [property: string]: any; -} - -/** - * The source of the change. This will change based on the context of the change (example: - * manual vs programmatic) - */ -export enum ChangeSource { - Automated = "Automated", - Derived = "Derived", - Ingested = "Ingested", - Manual = "Manual", - Propagated = "Propagated", - Suggested = "Suggested", -} - -export interface FieldChange { - /** - * Name of the entity field that changed. - */ - name?: string; - /** - * New value of the field. Note that this is a JSON string and use the corresponding field - * type to deserialize it. - */ - newValue?: any; - /** - * Previous value of the field. Note that this is a JSON string and use the corresponding - * field type to deserialize it. - */ - oldValue?: any; -} - -/** - * OpenMetadata Connection Config - */ -export interface OpenMetadataConnection { - /** - * OpenMetadata server API version to use. - */ - apiVersion?: string; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - */ - hostPort: string; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: Config; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: OpenmetadataType; - /** - * Flag to verify SSL Certificate for OpenMetadata Server. - */ - verifySSL?: VerifySSL; -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * SSL Configuration for OpenMetadata Server - * - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface Config { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Service Type - * - * OpenMetadata service type - */ -export enum OpenmetadataType { - OpenMetadata = "OpenMetadata", -} - -/** - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * Permission used by Native Applications. - * - * This schema defines the Permission used by Native Application. - */ -export enum Permissions { - All = "All", -} - -/** - * Application Private configuration loaded at runtime. - * - * Private Configuration for the CollateAI External Application. - */ -export interface PrivateConfig { - /** - * Collate Server public URL. WAII will use this information to interact with the server. - * E.g., https://sandbox.getcollate.io - */ - collateURL?: string; - /** - * Limits for the CollateAI Application. - */ - limits?: AppLimitsConfig; - /** - * WAII API Token - */ - token?: string; - /** - * WAII API host URL - */ - waiiInstance?: string; - [property: string]: any; -} - -/** - * Limits for the CollateAI Application. - * - * Private Configuration for the App Limits. - */ -export interface AppLimitsConfig { - /** - * The records of the limits. - */ - actions: { [key: string]: number }; - /** - * The start of this limit cycle. - */ - billingCycleStart: Date; -} - -/** - * Type of provider of an entity. Some entities are provided by the `system`. Some are - * entities created and provided by the `user`. Typically `system` provide entities can't be - * deleted and can only be disabled. Some apps such as AutoPilot create entities with - * `automation` provider type. These entities can be deleted by the user. - */ -export enum ProviderType { - Automation = "automation", - System = "system", - User = "user", -} - -/** - * Execution Configuration. - * - * Live Execution object. - * - * Scheduled Execution Context Configuration. - */ -export interface ExecutionContext { -} - -/** - * This schema defines the Schedule Type of Application. - * - * This schema defines the type of application. - */ -export enum ScheduleType { - Live = "Live", - NoSchedule = "NoSchedule", - OnlyManual = "OnlyManual", - Scheduled = "Scheduled", - ScheduledOrManual = "ScheduledOrManual", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/slackAppTokenConfiguration.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/slackAppTokenConfiguration.ts deleted file mode 100644 index 00933526826..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/external/slackAppTokenConfiguration.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This schema defines the Slack App Token Configuration - */ -export interface SlackAppTokenConfiguration { - /** - * Bot Token - */ - botToken: string; - /** - * User Token - */ - userToken: string; -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts deleted file mode 100644 index 667bab0b5d6..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts +++ /dev/null @@ -1,1643 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This schema defines the applications for Open-Metadata. - */ -export interface AppMarketPlaceDefinition { - /** - * This schema defines the type of the agent. - */ - agentType?: AgentType; - /** - * If true, multiple instances of this app can run concurrently. This is useful for apps - * like QueryRunner that support parallel executions with different configurations. - */ - allowConcurrentExecution?: boolean; - /** - * Allow users to configure the app from the UI. If `false`, the `configure` step will be - * hidden. - */ - allowConfiguration?: boolean; - /** - * Application Configuration object. - */ - appConfiguration?: any[] | boolean | number | null | CollateAIAppConfig | string; - /** - * Application Logo Url. - */ - appLogoUrl?: string; - /** - * Application Screenshots. - */ - appScreenshots?: string[]; - /** - * This schema defines the type of application. - */ - appType: AppType; - /** - * Change that lead to this version of the entity. - */ - changeDescription?: ChangeDescription; - /** - * Full Qualified ClassName for the the application. Use can use - * 'org.openmetadata.service.apps.AbstractNativeApplication' if you don't have one yet. - */ - className: string; - /** - * When `true` indicates the entity has been soft deleted. - */ - deleted?: boolean; - /** - * Description of the Application. - */ - description?: string; - /** - * Developer For the Application. - */ - developer?: string; - /** - * Url for the developer - */ - developerUrl?: string; - /** - * Display Name for the application. - */ - displayName?: string; - /** - * Domains the asset belongs to. When not set, the asset inherits the domain from the parent - * it belongs to. - */ - domains?: EntityReference[]; - /** - * Event subscriptions that will be created when the application is installed. - */ - eventSubscriptions?: CreateEventSubscription[]; - /** - * Features of the Application. - */ - features?: string; - /** - * FullyQualifiedName same as `name`. - */ - fullyQualifiedName?: string; - /** - * Link to the resource corresponding to this entity. - */ - href?: string; - /** - * Unique identifier of this application. - */ - id: string; - /** - * Bot user that performed the action on behalf of the actual user. - */ - impersonatedBy?: string; - /** - * Change that lead to this version of the entity. - */ - incrementalChangeDescription?: ChangeDescription; - /** - * Name of the Application. - */ - name: string; - /** - * Owners of this workflow. - */ - owners?: EntityReference[]; - /** - * Permission used by Native Applications. - */ - permission: Permissions; - /** - * Flag to enable/disable preview for the application. If the app is in preview mode, it - * can't be installed. - */ - preview?: boolean; - /** - * Privacy Policy for the developer - */ - privacyPolicyUrl?: string; - /** - * If app type is live, user can provide additional runtime context. - */ - runtime?: ExecutionContext; - /** - * This schema defines the Schedule Type of Application. - */ - scheduleType: ScheduleType; - /** - * Fully Qualified class name for the Python source that will execute the external - * application. - */ - sourcePythonClass?: string; - /** - * Support Email for the application - */ - supportEmail?: string; - /** - * If the app support execution through the external runner. - */ - supportsIngestionRunner?: boolean; - /** - * If the app run can be interrupted as part of the execution. - */ - supportsInterrupt?: boolean; - /** - * A system app cannot be uninstalled or modified. - */ - system?: boolean; - /** - * Tags associated with the entity. - */ - tags?: TagElement[]; - /** - * Last update time corresponding to the new version of the entity in Unix epoch time - * milliseconds. - */ - updatedAt?: number; - /** - * User who made the update. - */ - updatedBy?: string; - /** - * Metadata version of the entity. - */ - version?: number; -} - -/** - * This schema defines the type of the agent. - * - * This schema defines the type of application. - */ -export enum AgentType { - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - CollateAITierAgent = "CollateAITierAgent", - Metadata = "Metadata", -} - -/** - * Configuration for the CollateAI External Application. - * - * Configuration for the Automator External Application. - * - * This schema defines the Slack App Token Configuration - * - * Configuration for the Collate AI Quality Agent. - * - * No configuration needed to instantiate the Data Insights Pipeline. The logic is handled - * in the backend. - * - * Search Indexing App. - * - * Cache Warmup Application Configuration. - * - * Configuration for the AutoPilot Application. - */ -export interface CollateAIAppConfig { - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - filter?: string; - /** - * Patch the description if it is empty, instead of raising a suggestion - * - * Patch the tier if it is empty, instead of raising a suggestion - */ - patchIfEmpty?: boolean; - /** - * Application Type - */ - type?: Type; - /** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - */ - actions?: Action[]; - /** - * Entities selected to run the automation. - */ - resources?: Resource; - /** - * Bot Token - */ - botToken?: string; - /** - * User Token - */ - userToken?: string; - /** - * Whether the suggested tests should be active or not upon suggestion - * - * Whether the AutoPilot Workflow should be active or not. - */ - active?: boolean; - backfillConfiguration?: BackfillConfiguration; - /** - * Maximum number of events processed at a time (Default 100). - * - * Maximum number of events sent in a batch (Default 100). - * - * Number of entities to process in each batch. - */ - batchSize?: number; - moduleConfiguration?: ModuleConfiguration; - /** - * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property - * Type and are facing errors. Bear in mind that recreating the index will delete your - * DataAssets and a backfill will be needed. - */ - recreateDataAssetsIndex?: boolean; - sendToAdmins?: boolean; - sendToTeams?: boolean; - /** - * Enable automatic performance tuning based on cluster capabilities and database entity - * count - */ - autoTune?: boolean; - /** - * Number of threads to use for reindexing - * - * Number of parallel threads for processing entities and warming cache. - */ - consumerThreads?: number; - /** - * List of Entities to Reindex - * - * List of entity types to warm up in cache. Use 'all' to warm up all entity types. - */ - entities?: string[]; - /** - * Initial backoff time in milliseconds - */ - initialBackoff?: number; - /** - * Maximum backoff time in milliseconds - */ - maxBackoff?: number; - /** - * Maximum number of concurrent requests to the search index - */ - maxConcurrentRequests?: number; - /** - * Maximum number of retries for a failed request - */ - maxRetries?: number; - /** - * Maximum number of events sent in a batch (Default 100). - */ - payLoadSize?: number; - /** - * Number of threads to use for reindexing - */ - producerThreads?: number; - /** - * Queue Size to user internally for reindexing. - * - * Internal queue size for entity processing pipeline. - */ - queueSize?: number; - /** - * This schema publisher run modes. - */ - recreateIndex?: boolean; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Force cache warmup even if another instance is detected (use with caution). - */ - force?: boolean; - /** - * Enter the retention period for Activity Threads of type = 'Conversation' records in days - * (e.g., 30 for one month, 60 for two months). - */ - activityThreadsRetentionPeriod?: number; - /** - * Enter the retention period for change event records in days (e.g., 7 for one week, 30 for - * one month). - */ - changeEventRetentionPeriod?: number; - /** - * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two - * months). - */ - profileDataRetentionPeriod?: number; - /** - * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for - * two months). - */ - testCaseResultsRetentionPeriod?: number; - /** - * Service Entity Link for which to trigger the application. - */ - entityLink?: string; - [property: string]: any; -} - -/** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - * - * Apply Classification Tags to the selected assets. - * - * Remove Classification Tags Action Type - * - * Apply Glossary Terms to the selected assets. - * - * Remove Glossary Terms Action Type - * - * Add domains to the selected assets. - * - * Remove domains from the selected assets. - * - * Apply Tags to the selected assets. - * - * Add a Custom Property to the selected assets. - * - * Remove Owner Action Type - * - * Add an owner to the selected assets. - * - * Add Test Cases to the selected assets. - * - * Remove Test Cases Action Type - * - * Add owners to the selected assets. - * - * Remove Custom Properties Action Type - * - * Add a Data Product to the selected assets. - * - * Remove a Data Product to the selected assets. - * - * Propagate description, tags and glossary terms via lineage - * - * ML Tagging action configuration for external automator. - */ -export interface Action { - /** - * Apply tags to the children of the selected assets that match the criteria. E.g., columns, - * tasks, topic fields,... - * - * Remove tags from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply terms to the children of the selected assets that match the criteria. E.g., - * columns, tasks, topic fields,... - * - * Remove terms from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply the description to the children of the selected assets that match the criteria. - * E.g., columns, tasks, topic fields,... - * - * Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Add tests to the selected table columns - * - * Remove tests to the selected table columns - */ - applyToChildren?: string[]; - /** - * Update tags even if they are already defined in the asset. By default, incoming tags are - * merged with the existing ones. - * - * Update terms even if they are already defined in the asset. By default, incoming terms - * are merged with the existing ones. - * - * Update the domains even if they are defined in the asset. By default, we will only apply - * the domains to assets without domains. - * - * Update the description even if they are already defined in the asset. By default, we'll - * only add the descriptions to assets without the description set. - * - * Update the Custom Property even if it is defined in the asset. By default, we will only - * apply the owners to assets without the given Custom Property informed. - * - * Update the tier even if it is defined in the asset. By default, we will only apply the - * tier to assets without tier. - * - * Update the test even if it is defined in the asset. By default, we will only apply the - * test to assets without the existing test already existing. - * - * Update the owners even if it is defined in the asset. By default, we will only apply the - * owners to assets without owner. - * - * Update the Data Product even if the asset belongs to a different Domain. By default, we - * will only add the Data Product if the asset has no Domain, or it belongs to the same - * domain as the Data Product. - * - * Update descriptions, tags and Glossary Terms via lineage even if they are already defined - * in the asset. By default, descriptions are only updated if they are not already defined - * in the asset, and incoming tags are merged with the existing ones. - */ - overwriteMetadata?: boolean; - /** - * Classification Tags to apply (source must be 'Classification') - * - * Classification Tags to remove (source must be 'Classification') - */ - tags?: TagElement[]; - /** - * Application Type - */ - type: ActionType; - /** - * Remove tags from all the children and parent of the selected assets. - * - * Remove terms from all the children and parent of the selected assets. - * - * Remove descriptions from all the children and parent of the selected assets. - */ - applyToAll?: boolean; - /** - * Remove tags by its label type - * - * Remove terms by its label type - */ - labels?: LabelElement[]; - /** - * Glossary Terms to apply - * - * Glossary Terms to remove - */ - terms?: TagElement[]; - /** - * Domains to apply - */ - domains?: EntityReference[]; - /** - * Description to apply - */ - description?: string; - /** - * Owners to apply - * - * Custom Properties keys to remove - */ - customProperties?: any; - /** - * tier to apply - */ - tier?: TagElement; - /** - * Test Cases to apply - */ - testCases?: TestCaseDefinitions[]; - /** - * Remove all test cases - */ - removeAll?: boolean; - /** - * Test Cases to remove - */ - testCaseDefinitions?: string[]; - /** - * Owners to apply - */ - owners?: EntityReference[]; - /** - * Data Products to apply - * - * Data Products to remove - */ - dataProducts?: EntityReference[]; - /** - * Propagate the metadata to columns via column-level lineage. - */ - propagateColumnLevel?: boolean; - /** - * Propagate description through lineage - */ - propagateDescription?: boolean; - /** - * Propagate domains from the parent through lineage - */ - propagateDomains?: boolean; - /** - * Propagate glossary terms through lineage - */ - propagateGlossaryTerms?: boolean; - /** - * Propagate owner from the parent - */ - propagateOwner?: boolean; - /** - * Propagate the metadata to the parents (e.g., tables) via lineage. - */ - propagateParent?: boolean; - /** - * Propagate tags through lineage - */ - propagateTags?: boolean; - /** - * Propagate tier from the parent - */ - propagateTier?: boolean; - /** - * Number of levels to propagate lineage. If not set, it will propagate to all levels. - */ - propagationDepth?: number; - /** - * List of configurations to stop propagation based on conditions - */ - propagationStopConfigs?: PropagationStopConfig[]; -} - -/** - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Optional custom notification template for this subscription. When not set, system default - * template will be used. Only USER templates can be assigned. - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Remove tags by its label type - * - * Remove terms by its label type - */ -export enum LabelElement { - Automated = "Automated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Configuration to stop lineage propagation based on conditions - */ -export interface PropagationStopConfig { - /** - * The metadata attribute to check for stopping propagation - */ - metadataAttribute: MetadataAttribute; - /** - * List of attribute values that will stop propagation when any of them is matched - */ - value: Array; -} - -/** - * The metadata attribute to check for stopping propagation - */ -export enum MetadataAttribute { - Description = "description", - Domains = "domains", - GlossaryTerms = "glossaryTerms", - Owner = "owner", - Tags = "tags", - Tier = "tier", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - * - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Optional custom notification template for this subscription. When not set, system default - * template will be used. Only USER templates can be assigned. - */ -export interface TagLabel { - /** - * Description for the tag label. - * - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this tag. - * - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Link to the tag resource. - * - * Link to the entity resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType?: LabelTypeEnum; - /** - * Name of the tag or glossary term. - * - * Name of the entity instance. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source?: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state?: State; - style?: Style; - tagFQN?: string; - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id?: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type?: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelTypeEnum { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * Cover image configuration for the entity. - */ - coverImage?: CoverImage; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * Cover image configuration for the entity. - * - * Cover image configuration for an entity. This is used to display a banner or header image - * for entities like Domain, Glossary, Data Product, etc. - */ -export interface CoverImage { - /** - * Position of the cover image in CSS background-position format. Supports keywords (top, - * center, bottom) or pixel values (e.g., '20px 30px'). - */ - position?: string; - /** - * URL of the cover image. - */ - url?: string; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - */ -export interface TagElement { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelTypeEnum; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Minimum set of requirements to get a Test Case request ready - */ -export interface TestCaseDefinitions { - /** - * Compute the passed and failed row count for the test case. - */ - computePassedFailedRowCount?: boolean; - parameterValues?: TestCaseParameterValue[]; - /** - * Tags to apply - */ - tags?: TagElement[]; - /** - * Fully qualified name of the test definition. - */ - testDefinition?: string; - /** - * If the test definition supports it, use dynamic assertion to evaluate the test case. - */ - useDynamicAssertion?: boolean; - [property: string]: any; -} - -/** - * This schema defines the parameter values that can be passed for a Test Case. - */ -export interface TestCaseParameterValue { - /** - * name of the parameter. Must match the parameter names in testCaseParameterDefinition - */ - name?: string; - /** - * value to be passed for the Parameters. These are input from Users. We capture this in - * string and convert during the runtime. - */ - value?: string; - [property: string]: any; -} - -/** - * Application Type - * - * Add Tags action type. - * - * Remove Classification Tags Action Type. - * - * Add Terms action type. - * - * Remove Terms Action Type. - * - * Add Domain Action Type. - * - * Remove Domain Action Type - * - * Add Description Action Type. - * - * Add Custom Properties Action Type. - * - * Remove Description Action Type - * - * Add Tier Action Type. - * - * Remove Tier Action Type - * - * Add Test Case Action Type. - * - * Remove Test Case Action Type - * - * Add Owner Action Type. - * - * Remove Owner Action Type - * - * Remove Custom Properties Action Type. - * - * Add Data Products Action Type. - * - * Remove Data Products Action Type. - * - * Lineage propagation action type. - * - * ML PII Tagging action type. - */ -export enum ActionType { - AddCustomPropertiesAction = "AddCustomPropertiesAction", - AddDataProductAction = "AddDataProductAction", - AddDescriptionAction = "AddDescriptionAction", - AddDomainAction = "AddDomainAction", - AddOwnerAction = "AddOwnerAction", - AddTagsAction = "AddTagsAction", - AddTermsAction = "AddTermsAction", - AddTestCaseAction = "AddTestCaseAction", - AddTierAction = "AddTierAction", - LineagePropagationAction = "LineagePropagationAction", - MLTaggingAction = "MLTaggingAction", - RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", - RemoveDataProductAction = "RemoveDataProductAction", - RemoveDescriptionAction = "RemoveDescriptionAction", - RemoveDomainAction = "RemoveDomainAction", - RemoveOwnerAction = "RemoveOwnerAction", - RemoveTagsAction = "RemoveTagsAction", - RemoveTermsAction = "RemoveTermsAction", - RemoveTestCaseAction = "RemoveTestCaseAction", - RemoveTierAction = "RemoveTierAction", -} - -/** - * Backfill Configuration - */ -export interface BackfillConfiguration { - /** - * Enable Backfill for the configured dates - */ - enabled?: boolean; - /** - * Date for which the backfill will end - */ - endDate?: Date; - /** - * Date from which to start the backfill - */ - startDate?: Date; - [property: string]: any; -} - -/** - * Different Module Configurations - */ -export interface ModuleConfiguration { - /** - * App Analytics Module configuration - */ - appAnalytics: AppAnalyticsConfig; - /** - * Cost Analysis Insights Module configuration - */ - costAnalysis: CostAnalysisConfig; - /** - * Data Assets Insights Module configuration - */ - dataAssets: DataAssetsConfig; - /** - * Data Quality Insights Module configuration - */ - dataQuality: DataQualityConfig; -} - -/** - * App Analytics Module configuration - */ -export interface AppAnalyticsConfig { - /** - * If Enabled, App Analytics insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Cost Analysis Insights Module configuration - */ -export interface CostAnalysisConfig { - /** - * If Enabled, Cost Analysis insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Data Assets Insights Module configuration - */ -export interface DataAssetsConfig { - /** - * If Enabled, Data Asset insights will be populated when the App runs. - */ - enabled: boolean; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Defines the number of days the Data Assets Insights information will be kept. After it - * they will be deleted. - */ - retention?: number; - serviceFilter?: ServiceFilter; -} - -export interface ServiceFilter { - serviceName?: string; - serviceType?: string; -} - -/** - * Data Quality Insights Module configuration - */ -export interface DataQualityConfig { - /** - * If Enabled, Data Quality insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Entities selected to run the automation. - */ -export interface Resource { - /** - * Filter JSON tree to be used for rendering the filters in the UI. This comes from - * Immutable Tree type of react-awesome-query-builder. - */ - filterJsonTree?: string; - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - queryFilter?: string; - /** - * Type of the entity. E.g., 'table', 'chart',... - */ - type?: string[]; - [property: string]: any; -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Application Type - * - * Application type. - */ -export enum Type { - AutoPilotApplication = "AutoPilotApplication", - Automator = "Automator", - CacheWarmup = "CacheWarmup", - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - CollateAITierAgent = "CollateAITierAgent", - DataInsights = "DataInsights", - DataInsightsReport = "DataInsightsReport", - SearchIndexing = "SearchIndexing", -} - -/** - * This schema defines the type of application. - */ -export enum AppType { - External = "external", - Internal = "internal", -} - -/** - * Change that lead to this version of the entity. - * - * Description of the change. - */ -export interface ChangeDescription { - changeSummary?: { [key: string]: ChangeSummary }; - /** - * Names of fields added during the version changes. - */ - fieldsAdded?: FieldChange[]; - /** - * Fields deleted during the version changes with old value before deleted. - */ - fieldsDeleted?: FieldChange[]; - /** - * Fields modified during the version changes with old and new values. - */ - fieldsUpdated?: FieldChange[]; - /** - * When a change did not result in change, this could be same as the current version. - */ - previousVersion?: number; -} - -export interface ChangeSummary { - changedAt?: number; - /** - * Name of the user or bot who made this change - */ - changedBy?: string; - changeSource?: ChangeSource; - [property: string]: any; -} - -/** - * The source of the change. This will change based on the context of the change (example: - * manual vs programmatic) - */ -export enum ChangeSource { - Automated = "Automated", - Derived = "Derived", - Ingested = "Ingested", - Manual = "Manual", - Propagated = "Propagated", - Suggested = "Suggested", -} - -export interface FieldChange { - /** - * Name of the entity field that changed. - */ - name?: string; - /** - * New value of the field. Note that this is a JSON string and use the corresponding field - * type to deserialize it. - */ - newValue?: any; - /** - * Previous value of the field. Note that this is a JSON string and use the corresponding - * field type to deserialize it. - */ - oldValue?: any; -} - -/** - * This defines schema for sending alerts for OpenMetadata - */ -export interface CreateEventSubscription { - /** - * Type of Alert - */ - alertType: AlertType; - /** - * Maximum number of events sent in a batch (Default 10). - */ - batchSize?: number; - /** - * Consumer Class for the Event Subscription. Will use 'AlertPublisher' if not provided. - */ - className?: string; - config?: { [key: string]: any }; - /** - * A short description of the Alert, comprehensible to regular users. - */ - description?: string; - /** - * Subscription Config. - */ - destinations?: Destination[]; - /** - * Display name for this Alert. - */ - displayName?: string; - /** - * Fully qualified names of the domains the Event Subscription belongs to. - */ - domains?: string[]; - /** - * Is the alert enabled. - */ - enabled?: boolean; - /** - * Input for the Filters. - */ - input?: AlertFilteringInput; - /** - * Name that uniquely identifies this Alert. - */ - name: string; - /** - * Optional custom notification template for this subscription. When not set, system default - * template will be used. Only USER templates can be assigned. - */ - notificationTemplate?: EntityReference; - /** - * Owners of this Alert. - */ - owners?: EntityReference[]; - /** - * Poll Interval in seconds. - */ - pollInterval?: number; - provider?: ProviderType; - /** - * Defines a list of resources that triggers the Event Subscription, Eg All, User, Teams etc. - */ - resources?: string[]; - /** - * Number of times to retry callback on failure. (Default 3). - */ - retries?: number; - trigger?: Trigger; -} - -/** - * Type of Alert - * - * Type of Alerts supported. - */ -export enum AlertType { - ActivityFeed = "ActivityFeed", - Custom = "Custom", - GovernanceWorkflowChangeEvent = "GovernanceWorkflowChangeEvent", - Notification = "Notification", - Observability = "Observability", -} - -/** - * Subscription which has a type and the config. - */ -export interface Destination { - category: SubscriptionCategory; - config?: Webhook; - /** - * Maximum depth for downstream stakeholder notification traversal. If null, traverses - * without depth limit (with cycle protection). - */ - downstreamDepth?: number | null; - /** - * Is the subscription enabled. - */ - enabled?: boolean; - /** - * Unique identifier that identifies this Event Subscription. - */ - id?: string; - /** - * Enable notification of downstream entity stakeholders. When true, notifications will - * traverse lineage to include stakeholders of entities that consume data from the affected - * entity. - */ - notifyDownstream?: boolean; - /** - * Read timeout in seconds. (Default 12s). - */ - readTimeout?: number; - statusDetails?: TionStatus; - /** - * Connection timeout in seconds. (Default 10s). - */ - timeout?: number; - type: SubscriptionType; -} - -/** - * Subscription Endpoint Type. - */ -export enum SubscriptionCategory { - Admins = "Admins", - Assignees = "Assignees", - External = "External", - Followers = "Followers", - Mentions = "Mentions", - Owners = "Owners", - Teams = "Teams", - Users = "Users", -} - -/** - * This schema defines webhook for receiving events from OpenMetadata. - * - * This schema defines email config for receiving events from OpenMetadata. - * - * A generic map that can be deserialized later. - */ -export interface Webhook { - /** - * Endpoint to receive the webhook events over POST requests. - */ - endpoint?: string; - /** - * Custom headers to be sent with the webhook request. - */ - headers?: { [key: string]: any }; - /** - * HTTP operation to send the webhook request. Supports POST or PUT. - */ - httpMethod?: HTTPMethod; - /** - * Query parameters to be added to the webhook request URL. - */ - queryParams?: { [key: string]: any }; - /** - * List of receivers to send mail to - */ - receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; - /** - * Send the Event to Admins - * - * Send the Mails to Admins - */ - sendToAdmins?: boolean; - /** - * Send the Event to Followers - * - * Send the Mails to Followers - */ - sendToFollowers?: boolean; - /** - * Send the Event to Owners - * - * Send the Mails to Owners - */ - sendToOwners?: boolean; - [property: string]: any; -} - -/** - * HTTP operation to send the webhook request. Supports POST or PUT. - */ -export enum HTTPMethod { - Post = "POST", - Put = "PUT", -} - -/** - * Current status of the subscription, including details on the last successful and failed - * attempts, and retry information. - * - * Detailed status of the destination during a test operation, including HTTP response - * information. - */ -export interface TionStatus { - /** - * Timestamp of the last failed callback in UNIX UTC epoch time in milliseconds. - */ - lastFailedAt?: number; - /** - * Detailed reason for the last failure received during callback. - */ - lastFailedReason?: string; - /** - * HTTP status code received during the last failed callback attempt. - */ - lastFailedStatusCode?: number; - /** - * Timestamp of the last successful callback in UNIX UTC epoch time in milliseconds. - */ - lastSuccessfulAt?: number; - /** - * Timestamp for the next retry attempt in UNIX epoch time in milliseconds. Only valid if - * `status` is `awaitingRetry`. - */ - nextAttempt?: number; - /** - * Status is `disabled` when the event subscription was created with `enabled` set to false - * and it never started publishing events. Status is `active` when the event subscription is - * functioning normally and a 200 OK response was received for the callback notification. - * Status is `failed` when a bad callback URL, connection failures, or `1xx` or `3xx` - * response was received for the callback notification. Status is `awaitingRetry` when the - * previous attempt at callback timed out or received a `4xx` or `5xx` response. Status is - * `retryLimitReached` after all retries fail. - * - * Overall test status, indicating if the test operation succeeded or failed. - */ - status?: Status; - /** - * Current timestamp of this status in UNIX epoch time in milliseconds. - * - * Timestamp when the response was received, in UNIX epoch time milliseconds. - */ - timestamp?: number; - /** - * Body of the HTTP response, if any, returned by the server. - */ - entity?: string; - /** - * HTTP headers returned in the response as a map of header names to values. - */ - headers?: any; - /** - * URL location if the response indicates a redirect or newly created resource. - */ - location?: string; - /** - * Media type of the response entity, if specified (e.g., application/json). - */ - mediaType?: string; - /** - * Detailed reason for failure if the test did not succeed. - */ - reason?: string; - /** - * HTTP status code of the response (e.g., 200 for OK, 404 for Not Found). - */ - statusCode?: number; - /** - * HTTP status reason phrase associated with the status code (e.g., 'Not Found'). - */ - statusInfo?: string; -} - -/** - * Status is `disabled` when the event subscription was created with `enabled` set to false - * and it never started publishing events. Status is `active` when the event subscription is - * functioning normally and a 200 OK response was received for the callback notification. - * Status is `failed` when a bad callback URL, connection failures, or `1xx` or `3xx` - * response was received for the callback notification. Status is `awaitingRetry` when the - * previous attempt at callback timed out or received a `4xx` or `5xx` response. Status is - * `retryLimitReached` after all retries fail. - * - * Overall test status, indicating if the test operation succeeded or failed. - */ -export enum Status { - Active = "active", - AwaitingRetry = "awaitingRetry", - Disabled = "disabled", - Failed = "failed", - RetryLimitReached = "retryLimitReached", - StatusFailed = "Failed", - Success = "Success", -} - -/** - * Subscription Endpoint Type. - */ -export enum SubscriptionType { - ActivityFeed = "ActivityFeed", - Email = "Email", - GChat = "GChat", - GovernanceWorkflowChangeEvent = "GovernanceWorkflowChangeEvent", - MSTeams = "MsTeams", - Slack = "Slack", - Webhook = "Webhook", -} - -/** - * Input for the Filters. - * - * Observability of the event subscription. - */ -export interface AlertFilteringInput { - /** - * List of filters for the event subscription. - */ - actions?: ArgumentsInput[]; - /** - * List of filters for the event subscription. - */ - filters?: ArgumentsInput[]; -} - -/** - * Observability Filters for Event Subscription. - */ -export interface ArgumentsInput { - /** - * Arguments List - */ - arguments?: Argument[]; - effect?: Effect; - /** - * Name of the filter - */ - name?: string; - /** - * Prefix Condition for the filter. - */ - prefixCondition?: PrefixCondition; -} - -/** - * Argument for the filter. - */ -export interface Argument { - /** - * Value of the Argument - */ - input?: string[]; - /** - * Name of the Argument - */ - name?: string; -} - -export enum Effect { - Exclude = "exclude", - Include = "include", -} - -/** - * Prefix Condition for the filter. - * - * Prefix Condition to be applied to the Condition. - */ -export enum PrefixCondition { - And = "AND", - Or = "OR", -} - -/** - * Type of provider of an entity. Some entities are provided by the `system`. Some are - * entities created and provided by the `user`. Typically `system` provide entities can't be - * deleted and can only be disabled. Some apps such as AutoPilot create entities with - * `automation` provider type. These entities can be deleted by the user. - */ -export enum ProviderType { - Automation = "automation", - System = "system", - User = "user", -} - -/** - * Trigger Configuration for Alerts. - */ -export interface Trigger { - /** - * Cron Expression in case of Custom scheduled Trigger - */ - cronExpression?: string; - /** - * Schedule Info - */ - scheduleInfo?: ScheduleInfo; - triggerType: TriggerType; -} - -/** - * Schedule Info - */ -export enum ScheduleInfo { - Custom = "Custom", - Daily = "Daily", - Monthly = "Monthly", - Weekly = "Weekly", -} - -/** - * Trigger Configuration for Alerts. - */ -export enum TriggerType { - RealTime = "RealTime", - Scheduled = "Scheduled", -} - -/** - * Permission used by Native Applications. - * - * This schema defines the Permission used by Native Application. - */ -export enum Permissions { - All = "All", -} - -/** - * If app type is live, user can provide additional runtime context. - * - * Execution Configuration. - * - * Live Execution object. - * - * Scheduled Execution Context Configuration. - */ -export interface ExecutionContext { -} - -/** - * This schema defines the Schedule Type of Application. - * - * This schema defines the type of application. - */ -export enum ScheduleType { - Live = "Live", - NoSchedule = "NoSchedule", - OnlyManual = "OnlyManual", - Scheduled = "Scheduled", - ScheduledOrManual = "ScheduledOrManual", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts deleted file mode 100644 index c5e811c50cb..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts +++ /dev/null @@ -1,1535 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This schema defines the applications for Open-Metadata. - */ -export interface CreateAppMarketPlaceDefinitionReq { - /** - * This schema defines the type of the agent. - */ - agentType?: AgentType; - /** - * If true, multiple instances of this app can run concurrently. This is useful for apps - * like QueryRunner that support parallel executions with different configurations. - */ - allowConcurrentExecution?: boolean; - /** - * Allow users to configure the app from the UI. If `false`, the `configure` step will be - * hidden. - */ - allowConfiguration?: boolean; - /** - * Application Configuration object. - */ - appConfiguration?: any[] | boolean | number | null | CollateAIAppConfig | string; - /** - * Application Logo Url. - */ - appLogoUrl?: string; - /** - * Application Screenshots. - */ - appScreenshots?: string[]; - /** - * This schema defines the type of application. - */ - appType: AppType; - /** - * Full Qualified ClassName for the the application. Use can use - * 'org.openmetadata.service.apps.AbstractNativeApplication' if you don't have one yet. - */ - className: string; - /** - * Description of the Application. - */ - description?: string; - /** - * Developer For the Application. - */ - developer?: string; - /** - * Url for the developer - */ - developerUrl?: string; - /** - * Display Name for the application. - */ - displayName?: string; - /** - * Fully qualified names of the domains the Application belongs to. - */ - domains?: string[]; - /** - * The app will be installable only if this flag is set to true. - */ - enabled?: boolean; - /** - * Event subscriptions that will be created when the application is installed. - */ - eventSubscriptions?: CreateEventSubscription[]; - /** - * Features of the Application. - */ - features?: string; - /** - * Name of the Application. - */ - name: string; - /** - * Owners of this workflow. - */ - owners?: EntityReference[]; - /** - * Permission used by Native Applications. - */ - permission: Permissions; - /** - * Privacy Policy for the developer - */ - privacyPolicyUrl?: string; - /** - * If app type is live, user can provide additional runtime context. - */ - runtime?: ExecutionContext; - /** - * This schema defines the Schedule Type of Application. - */ - scheduleType: ScheduleType; - /** - * Fully Qualified class name for the Python source that will execute the external - * application. - */ - sourcePythonClass?: string; - /** - * Support Email for the application - */ - supportEmail?: string; - /** - * If the app support execution through the external runner. - */ - supportsIngestionRunner?: boolean; - /** - * If the app run can be interrupted as part of the execution. - */ - supportsInterrupt?: boolean; - /** - * A system app cannot be uninstalled or modified. - */ - system?: boolean; - /** - * Tags associated with the entity. - */ - tags?: TagElement[]; -} - -/** - * This schema defines the type of the agent. - * - * This schema defines the type of application. - */ -export enum AgentType { - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - CollateAITierAgent = "CollateAITierAgent", - Metadata = "Metadata", -} - -/** - * Configuration for the CollateAI External Application. - * - * Configuration for the Automator External Application. - * - * This schema defines the Slack App Token Configuration - * - * Configuration for the Collate AI Quality Agent. - * - * No configuration needed to instantiate the Data Insights Pipeline. The logic is handled - * in the backend. - * - * Search Indexing App. - * - * Cache Warmup Application Configuration. - * - * Configuration for the AutoPilot Application. - */ -export interface CollateAIAppConfig { - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - filter?: string; - /** - * Patch the description if it is empty, instead of raising a suggestion - * - * Patch the tier if it is empty, instead of raising a suggestion - */ - patchIfEmpty?: boolean; - /** - * Application Type - */ - type?: Type; - /** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - */ - actions?: Action[]; - /** - * Entities selected to run the automation. - */ - resources?: Resource; - /** - * Bot Token - */ - botToken?: string; - /** - * User Token - */ - userToken?: string; - /** - * Whether the suggested tests should be active or not upon suggestion - * - * Whether the AutoPilot Workflow should be active or not. - */ - active?: boolean; - backfillConfiguration?: BackfillConfiguration; - /** - * Maximum number of events processed at a time (Default 100). - * - * Maximum number of events sent in a batch (Default 100). - * - * Number of entities to process in each batch. - */ - batchSize?: number; - moduleConfiguration?: ModuleConfiguration; - /** - * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property - * Type and are facing errors. Bear in mind that recreating the index will delete your - * DataAssets and a backfill will be needed. - */ - recreateDataAssetsIndex?: boolean; - sendToAdmins?: boolean; - sendToTeams?: boolean; - /** - * Enable automatic performance tuning based on cluster capabilities and database entity - * count - */ - autoTune?: boolean; - /** - * Number of threads to use for reindexing - * - * Number of parallel threads for processing entities and warming cache. - */ - consumerThreads?: number; - /** - * List of Entities to Reindex - * - * List of entity types to warm up in cache. Use 'all' to warm up all entity types. - */ - entities?: string[]; - /** - * Initial backoff time in milliseconds - */ - initialBackoff?: number; - /** - * Maximum backoff time in milliseconds - */ - maxBackoff?: number; - /** - * Maximum number of concurrent requests to the search index - */ - maxConcurrentRequests?: number; - /** - * Maximum number of retries for a failed request - */ - maxRetries?: number; - /** - * Maximum number of events sent in a batch (Default 100). - */ - payLoadSize?: number; - /** - * Number of threads to use for reindexing - */ - producerThreads?: number; - /** - * Queue Size to user internally for reindexing. - * - * Internal queue size for entity processing pipeline. - */ - queueSize?: number; - /** - * This schema publisher run modes. - */ - recreateIndex?: boolean; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Force cache warmup even if another instance is detected (use with caution). - */ - force?: boolean; - /** - * Enter the retention period for Activity Threads of type = 'Conversation' records in days - * (e.g., 30 for one month, 60 for two months). - */ - activityThreadsRetentionPeriod?: number; - /** - * Enter the retention period for change event records in days (e.g., 7 for one week, 30 for - * one month). - */ - changeEventRetentionPeriod?: number; - /** - * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two - * months). - */ - profileDataRetentionPeriod?: number; - /** - * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for - * two months). - */ - testCaseResultsRetentionPeriod?: number; - /** - * Service Entity Link for which to trigger the application. - */ - entityLink?: string; - [property: string]: any; -} - -/** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - * - * Apply Classification Tags to the selected assets. - * - * Remove Classification Tags Action Type - * - * Apply Glossary Terms to the selected assets. - * - * Remove Glossary Terms Action Type - * - * Add domains to the selected assets. - * - * Remove domains from the selected assets. - * - * Apply Tags to the selected assets. - * - * Add a Custom Property to the selected assets. - * - * Remove Owner Action Type - * - * Add an owner to the selected assets. - * - * Add Test Cases to the selected assets. - * - * Remove Test Cases Action Type - * - * Add owners to the selected assets. - * - * Remove Custom Properties Action Type - * - * Add a Data Product to the selected assets. - * - * Remove a Data Product to the selected assets. - * - * Propagate description, tags and glossary terms via lineage - * - * ML Tagging action configuration for external automator. - */ -export interface Action { - /** - * Apply tags to the children of the selected assets that match the criteria. E.g., columns, - * tasks, topic fields,... - * - * Remove tags from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply terms to the children of the selected assets that match the criteria. E.g., - * columns, tasks, topic fields,... - * - * Remove terms from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply the description to the children of the selected assets that match the criteria. - * E.g., columns, tasks, topic fields,... - * - * Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Add tests to the selected table columns - * - * Remove tests to the selected table columns - */ - applyToChildren?: string[]; - /** - * Update tags even if they are already defined in the asset. By default, incoming tags are - * merged with the existing ones. - * - * Update terms even if they are already defined in the asset. By default, incoming terms - * are merged with the existing ones. - * - * Update the domains even if they are defined in the asset. By default, we will only apply - * the domains to assets without domains. - * - * Update the description even if they are already defined in the asset. By default, we'll - * only add the descriptions to assets without the description set. - * - * Update the Custom Property even if it is defined in the asset. By default, we will only - * apply the owners to assets without the given Custom Property informed. - * - * Update the tier even if it is defined in the asset. By default, we will only apply the - * tier to assets without tier. - * - * Update the test even if it is defined in the asset. By default, we will only apply the - * test to assets without the existing test already existing. - * - * Update the owners even if it is defined in the asset. By default, we will only apply the - * owners to assets without owner. - * - * Update the Data Product even if the asset belongs to a different Domain. By default, we - * will only add the Data Product if the asset has no Domain, or it belongs to the same - * domain as the Data Product. - * - * Update descriptions, tags and Glossary Terms via lineage even if they are already defined - * in the asset. By default, descriptions are only updated if they are not already defined - * in the asset, and incoming tags are merged with the existing ones. - */ - overwriteMetadata?: boolean; - /** - * Classification Tags to apply (source must be 'Classification') - * - * Classification Tags to remove (source must be 'Classification') - */ - tags?: TagElement[]; - /** - * Application Type - */ - type: ActionType; - /** - * Remove tags from all the children and parent of the selected assets. - * - * Remove terms from all the children and parent of the selected assets. - * - * Remove descriptions from all the children and parent of the selected assets. - */ - applyToAll?: boolean; - /** - * Remove tags by its label type - * - * Remove terms by its label type - */ - labels?: LabelElement[]; - /** - * Glossary Terms to apply - * - * Glossary Terms to remove - */ - terms?: TagElement[]; - /** - * Domains to apply - */ - domains?: EntityReference[]; - /** - * Description to apply - */ - description?: string; - /** - * Owners to apply - * - * Custom Properties keys to remove - */ - customProperties?: any; - /** - * tier to apply - */ - tier?: TagElement; - /** - * Test Cases to apply - */ - testCases?: TestCaseDefinitions[]; - /** - * Remove all test cases - */ - removeAll?: boolean; - /** - * Test Cases to remove - */ - testCaseDefinitions?: string[]; - /** - * Owners to apply - */ - owners?: EntityReference[]; - /** - * Data Products to apply - * - * Data Products to remove - */ - dataProducts?: EntityReference[]; - /** - * Propagate the metadata to columns via column-level lineage. - */ - propagateColumnLevel?: boolean; - /** - * Propagate description through lineage - */ - propagateDescription?: boolean; - /** - * Propagate domains from the parent through lineage - */ - propagateDomains?: boolean; - /** - * Propagate glossary terms through lineage - */ - propagateGlossaryTerms?: boolean; - /** - * Propagate owner from the parent - */ - propagateOwner?: boolean; - /** - * Propagate the metadata to the parents (e.g., tables) via lineage. - */ - propagateParent?: boolean; - /** - * Propagate tags through lineage - */ - propagateTags?: boolean; - /** - * Propagate tier from the parent - */ - propagateTier?: boolean; - /** - * Number of levels to propagate lineage. If not set, it will propagate to all levels. - */ - propagationDepth?: number; - /** - * List of configurations to stop propagation based on conditions - */ - propagationStopConfigs?: PropagationStopConfig[]; -} - -/** - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Optional custom notification template for this subscription. When not set, system default - * template will be used. Only USER templates can be assigned. - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Remove tags by its label type - * - * Remove terms by its label type - */ -export enum LabelElement { - Automated = "Automated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Configuration to stop lineage propagation based on conditions - */ -export interface PropagationStopConfig { - /** - * The metadata attribute to check for stopping propagation - */ - metadataAttribute: MetadataAttribute; - /** - * List of attribute values that will stop propagation when any of them is matched - */ - value: Array; -} - -/** - * The metadata attribute to check for stopping propagation - */ -export enum MetadataAttribute { - Description = "description", - Domains = "domains", - GlossaryTerms = "glossaryTerms", - Owner = "owner", - Tags = "tags", - Tier = "tier", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - * - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Optional custom notification template for this subscription. When not set, system default - * template will be used. Only USER templates can be assigned. - */ -export interface TagLabel { - /** - * Description for the tag label. - * - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this tag. - * - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Link to the tag resource. - * - * Link to the entity resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType?: LabelTypeEnum; - /** - * Name of the tag or glossary term. - * - * Name of the entity instance. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source?: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state?: State; - style?: Style; - tagFQN?: string; - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id?: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type?: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelTypeEnum { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * Cover image configuration for the entity. - */ - coverImage?: CoverImage; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * Cover image configuration for the entity. - * - * Cover image configuration for an entity. This is used to display a banner or header image - * for entities like Domain, Glossary, Data Product, etc. - */ -export interface CoverImage { - /** - * Position of the cover image in CSS background-position format. Supports keywords (top, - * center, bottom) or pixel values (e.g., '20px 30px'). - */ - position?: string; - /** - * URL of the cover image. - */ - url?: string; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - */ -export interface TagElement { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelTypeEnum; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Minimum set of requirements to get a Test Case request ready - */ -export interface TestCaseDefinitions { - /** - * Compute the passed and failed row count for the test case. - */ - computePassedFailedRowCount?: boolean; - parameterValues?: TestCaseParameterValue[]; - /** - * Tags to apply - */ - tags?: TagElement[]; - /** - * Fully qualified name of the test definition. - */ - testDefinition?: string; - /** - * If the test definition supports it, use dynamic assertion to evaluate the test case. - */ - useDynamicAssertion?: boolean; - [property: string]: any; -} - -/** - * This schema defines the parameter values that can be passed for a Test Case. - */ -export interface TestCaseParameterValue { - /** - * name of the parameter. Must match the parameter names in testCaseParameterDefinition - */ - name?: string; - /** - * value to be passed for the Parameters. These are input from Users. We capture this in - * string and convert during the runtime. - */ - value?: string; - [property: string]: any; -} - -/** - * Application Type - * - * Add Tags action type. - * - * Remove Classification Tags Action Type. - * - * Add Terms action type. - * - * Remove Terms Action Type. - * - * Add Domain Action Type. - * - * Remove Domain Action Type - * - * Add Description Action Type. - * - * Add Custom Properties Action Type. - * - * Remove Description Action Type - * - * Add Tier Action Type. - * - * Remove Tier Action Type - * - * Add Test Case Action Type. - * - * Remove Test Case Action Type - * - * Add Owner Action Type. - * - * Remove Owner Action Type - * - * Remove Custom Properties Action Type. - * - * Add Data Products Action Type. - * - * Remove Data Products Action Type. - * - * Lineage propagation action type. - * - * ML PII Tagging action type. - */ -export enum ActionType { - AddCustomPropertiesAction = "AddCustomPropertiesAction", - AddDataProductAction = "AddDataProductAction", - AddDescriptionAction = "AddDescriptionAction", - AddDomainAction = "AddDomainAction", - AddOwnerAction = "AddOwnerAction", - AddTagsAction = "AddTagsAction", - AddTermsAction = "AddTermsAction", - AddTestCaseAction = "AddTestCaseAction", - AddTierAction = "AddTierAction", - LineagePropagationAction = "LineagePropagationAction", - MLTaggingAction = "MLTaggingAction", - RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", - RemoveDataProductAction = "RemoveDataProductAction", - RemoveDescriptionAction = "RemoveDescriptionAction", - RemoveDomainAction = "RemoveDomainAction", - RemoveOwnerAction = "RemoveOwnerAction", - RemoveTagsAction = "RemoveTagsAction", - RemoveTermsAction = "RemoveTermsAction", - RemoveTestCaseAction = "RemoveTestCaseAction", - RemoveTierAction = "RemoveTierAction", -} - -/** - * Backfill Configuration - */ -export interface BackfillConfiguration { - /** - * Enable Backfill for the configured dates - */ - enabled?: boolean; - /** - * Date for which the backfill will end - */ - endDate?: Date; - /** - * Date from which to start the backfill - */ - startDate?: Date; - [property: string]: any; -} - -/** - * Different Module Configurations - */ -export interface ModuleConfiguration { - /** - * App Analytics Module configuration - */ - appAnalytics: AppAnalyticsConfig; - /** - * Cost Analysis Insights Module configuration - */ - costAnalysis: CostAnalysisConfig; - /** - * Data Assets Insights Module configuration - */ - dataAssets: DataAssetsConfig; - /** - * Data Quality Insights Module configuration - */ - dataQuality: DataQualityConfig; -} - -/** - * App Analytics Module configuration - */ -export interface AppAnalyticsConfig { - /** - * If Enabled, App Analytics insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Cost Analysis Insights Module configuration - */ -export interface CostAnalysisConfig { - /** - * If Enabled, Cost Analysis insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Data Assets Insights Module configuration - */ -export interface DataAssetsConfig { - /** - * If Enabled, Data Asset insights will be populated when the App runs. - */ - enabled: boolean; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Defines the number of days the Data Assets Insights information will be kept. After it - * they will be deleted. - */ - retention?: number; - serviceFilter?: ServiceFilter; -} - -export interface ServiceFilter { - serviceName?: string; - serviceType?: string; -} - -/** - * Data Quality Insights Module configuration - */ -export interface DataQualityConfig { - /** - * If Enabled, Data Quality insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Entities selected to run the automation. - */ -export interface Resource { - /** - * Filter JSON tree to be used for rendering the filters in the UI. This comes from - * Immutable Tree type of react-awesome-query-builder. - */ - filterJsonTree?: string; - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - queryFilter?: string; - /** - * Type of the entity. E.g., 'table', 'chart',... - */ - type?: string[]; - [property: string]: any; -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Application Type - * - * Application type. - */ -export enum Type { - AutoPilotApplication = "AutoPilotApplication", - Automator = "Automator", - CacheWarmup = "CacheWarmup", - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - CollateAITierAgent = "CollateAITierAgent", - DataInsights = "DataInsights", - DataInsightsReport = "DataInsightsReport", - SearchIndexing = "SearchIndexing", -} - -/** - * This schema defines the type of application. - */ -export enum AppType { - External = "external", - Internal = "internal", -} - -/** - * This defines schema for sending alerts for OpenMetadata - */ -export interface CreateEventSubscription { - /** - * Type of Alert - */ - alertType: AlertType; - /** - * Maximum number of events sent in a batch (Default 10). - */ - batchSize?: number; - /** - * Consumer Class for the Event Subscription. Will use 'AlertPublisher' if not provided. - */ - className?: string; - config?: { [key: string]: any }; - /** - * A short description of the Alert, comprehensible to regular users. - */ - description?: string; - /** - * Subscription Config. - */ - destinations?: Destination[]; - /** - * Display name for this Alert. - */ - displayName?: string; - /** - * Fully qualified names of the domains the Event Subscription belongs to. - */ - domains?: string[]; - /** - * Is the alert enabled. - */ - enabled?: boolean; - /** - * Input for the Filters. - */ - input?: AlertFilteringInput; - /** - * Name that uniquely identifies this Alert. - */ - name: string; - /** - * Optional custom notification template for this subscription. When not set, system default - * template will be used. Only USER templates can be assigned. - */ - notificationTemplate?: EntityReference; - /** - * Owners of this Alert. - */ - owners?: EntityReference[]; - /** - * Poll Interval in seconds. - */ - pollInterval?: number; - provider?: ProviderType; - /** - * Defines a list of resources that triggers the Event Subscription, Eg All, User, Teams etc. - */ - resources?: string[]; - /** - * Number of times to retry callback on failure. (Default 3). - */ - retries?: number; - trigger?: Trigger; -} - -/** - * Type of Alert - * - * Type of Alerts supported. - */ -export enum AlertType { - ActivityFeed = "ActivityFeed", - Custom = "Custom", - GovernanceWorkflowChangeEvent = "GovernanceWorkflowChangeEvent", - Notification = "Notification", - Observability = "Observability", -} - -/** - * Subscription which has a type and the config. - */ -export interface Destination { - category: SubscriptionCategory; - config?: Webhook; - /** - * Maximum depth for downstream stakeholder notification traversal. If null, traverses - * without depth limit (with cycle protection). - */ - downstreamDepth?: number | null; - /** - * Is the subscription enabled. - */ - enabled?: boolean; - /** - * Unique identifier that identifies this Event Subscription. - */ - id?: string; - /** - * Enable notification of downstream entity stakeholders. When true, notifications will - * traverse lineage to include stakeholders of entities that consume data from the affected - * entity. - */ - notifyDownstream?: boolean; - /** - * Read timeout in seconds. (Default 12s). - */ - readTimeout?: number; - statusDetails?: TionStatus; - /** - * Connection timeout in seconds. (Default 10s). - */ - timeout?: number; - type: SubscriptionType; -} - -/** - * Subscription Endpoint Type. - */ -export enum SubscriptionCategory { - Admins = "Admins", - Assignees = "Assignees", - External = "External", - Followers = "Followers", - Mentions = "Mentions", - Owners = "Owners", - Teams = "Teams", - Users = "Users", -} - -/** - * This schema defines webhook for receiving events from OpenMetadata. - * - * This schema defines email config for receiving events from OpenMetadata. - * - * A generic map that can be deserialized later. - */ -export interface Webhook { - /** - * Endpoint to receive the webhook events over POST requests. - */ - endpoint?: string; - /** - * Custom headers to be sent with the webhook request. - */ - headers?: { [key: string]: any }; - /** - * HTTP operation to send the webhook request. Supports POST or PUT. - */ - httpMethod?: HTTPMethod; - /** - * Query parameters to be added to the webhook request URL. - */ - queryParams?: { [key: string]: any }; - /** - * List of receivers to send mail to - */ - receivers?: string[]; - /** - * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook - * payload and sent in `X-OM-Signature` header in POST requests to publish the events. - */ - secretKey?: string; - /** - * Send the Event to Admins - * - * Send the Mails to Admins - */ - sendToAdmins?: boolean; - /** - * Send the Event to Followers - * - * Send the Mails to Followers - */ - sendToFollowers?: boolean; - /** - * Send the Event to Owners - * - * Send the Mails to Owners - */ - sendToOwners?: boolean; - [property: string]: any; -} - -/** - * HTTP operation to send the webhook request. Supports POST or PUT. - */ -export enum HTTPMethod { - Post = "POST", - Put = "PUT", -} - -/** - * Current status of the subscription, including details on the last successful and failed - * attempts, and retry information. - * - * Detailed status of the destination during a test operation, including HTTP response - * information. - */ -export interface TionStatus { - /** - * Timestamp of the last failed callback in UNIX UTC epoch time in milliseconds. - */ - lastFailedAt?: number; - /** - * Detailed reason for the last failure received during callback. - */ - lastFailedReason?: string; - /** - * HTTP status code received during the last failed callback attempt. - */ - lastFailedStatusCode?: number; - /** - * Timestamp of the last successful callback in UNIX UTC epoch time in milliseconds. - */ - lastSuccessfulAt?: number; - /** - * Timestamp for the next retry attempt in UNIX epoch time in milliseconds. Only valid if - * `status` is `awaitingRetry`. - */ - nextAttempt?: number; - /** - * Status is `disabled` when the event subscription was created with `enabled` set to false - * and it never started publishing events. Status is `active` when the event subscription is - * functioning normally and a 200 OK response was received for the callback notification. - * Status is `failed` when a bad callback URL, connection failures, or `1xx` or `3xx` - * response was received for the callback notification. Status is `awaitingRetry` when the - * previous attempt at callback timed out or received a `4xx` or `5xx` response. Status is - * `retryLimitReached` after all retries fail. - * - * Overall test status, indicating if the test operation succeeded or failed. - */ - status?: Status; - /** - * Current timestamp of this status in UNIX epoch time in milliseconds. - * - * Timestamp when the response was received, in UNIX epoch time milliseconds. - */ - timestamp?: number; - /** - * Body of the HTTP response, if any, returned by the server. - */ - entity?: string; - /** - * HTTP headers returned in the response as a map of header names to values. - */ - headers?: any; - /** - * URL location if the response indicates a redirect or newly created resource. - */ - location?: string; - /** - * Media type of the response entity, if specified (e.g., application/json). - */ - mediaType?: string; - /** - * Detailed reason for failure if the test did not succeed. - */ - reason?: string; - /** - * HTTP status code of the response (e.g., 200 for OK, 404 for Not Found). - */ - statusCode?: number; - /** - * HTTP status reason phrase associated with the status code (e.g., 'Not Found'). - */ - statusInfo?: string; -} - -/** - * Status is `disabled` when the event subscription was created with `enabled` set to false - * and it never started publishing events. Status is `active` when the event subscription is - * functioning normally and a 200 OK response was received for the callback notification. - * Status is `failed` when a bad callback URL, connection failures, or `1xx` or `3xx` - * response was received for the callback notification. Status is `awaitingRetry` when the - * previous attempt at callback timed out or received a `4xx` or `5xx` response. Status is - * `retryLimitReached` after all retries fail. - * - * Overall test status, indicating if the test operation succeeded or failed. - */ -export enum Status { - Active = "active", - AwaitingRetry = "awaitingRetry", - Disabled = "disabled", - Failed = "failed", - RetryLimitReached = "retryLimitReached", - StatusFailed = "Failed", - Success = "Success", -} - -/** - * Subscription Endpoint Type. - */ -export enum SubscriptionType { - ActivityFeed = "ActivityFeed", - Email = "Email", - GChat = "GChat", - GovernanceWorkflowChangeEvent = "GovernanceWorkflowChangeEvent", - MSTeams = "MsTeams", - Slack = "Slack", - Webhook = "Webhook", -} - -/** - * Input for the Filters. - * - * Observability of the event subscription. - */ -export interface AlertFilteringInput { - /** - * List of filters for the event subscription. - */ - actions?: ArgumentsInput[]; - /** - * List of filters for the event subscription. - */ - filters?: ArgumentsInput[]; -} - -/** - * Observability Filters for Event Subscription. - */ -export interface ArgumentsInput { - /** - * Arguments List - */ - arguments?: Argument[]; - effect?: Effect; - /** - * Name of the filter - */ - name?: string; - /** - * Prefix Condition for the filter. - */ - prefixCondition?: PrefixCondition; -} - -/** - * Argument for the filter. - */ -export interface Argument { - /** - * Value of the Argument - */ - input?: string[]; - /** - * Name of the Argument - */ - name?: string; -} - -export enum Effect { - Exclude = "exclude", - Include = "include", -} - -/** - * Prefix Condition for the filter. - * - * Prefix Condition to be applied to the Condition. - */ -export enum PrefixCondition { - And = "AND", - Or = "OR", -} - -/** - * Type of provider of an entity. Some entities are provided by the `system`. Some are - * entities created and provided by the `user`. Typically `system` provide entities can't be - * deleted and can only be disabled. Some apps such as AutoPilot create entities with - * `automation` provider type. These entities can be deleted by the user. - */ -export enum ProviderType { - Automation = "automation", - System = "system", - User = "user", -} - -/** - * Trigger Configuration for Alerts. - */ -export interface Trigger { - /** - * Cron Expression in case of Custom scheduled Trigger - */ - cronExpression?: string; - /** - * Schedule Info - */ - scheduleInfo?: ScheduleInfo; - triggerType: TriggerType; -} - -/** - * Schedule Info - */ -export enum ScheduleInfo { - Custom = "Custom", - Daily = "Daily", - Monthly = "Monthly", - Weekly = "Weekly", -} - -/** - * Trigger Configuration for Alerts. - */ -export enum TriggerType { - RealTime = "RealTime", - Scheduled = "Scheduled", -} - -/** - * Permission used by Native Applications. - * - * This schema defines the Permission used by Native Application. - */ -export enum Permissions { - All = "All", -} - -/** - * If app type is live, user can provide additional runtime context. - * - * Execution Configuration. - * - * Live Execution object. - * - * Scheduled Execution Context Configuration. - */ -export interface ExecutionContext { -} - -/** - * This schema defines the Schedule Type of Application. - * - * This schema defines the type of application. - */ -export enum ScheduleType { - Live = "Live", - NoSchedule = "NoSchedule", - OnlyManual = "OnlyManual", - Scheduled = "Scheduled", - ScheduledOrManual = "ScheduledOrManual", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts deleted file mode 100644 index 5dec5177715..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts +++ /dev/null @@ -1,6846 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Ingestion Pipeline Config is used to set up a DAG and deploy. This entity is used to - * setup metadata/quality pipelines on Apache Airflow. - */ -export interface IngestionPipeline { - airflowConfig: AirflowConfig; - /** - * Type of the application when pipelineType is 'application'. - */ - applicationType?: string; - /** - * Change that led to this version of the entity. - */ - changeDescription?: ChangeDescription; - /** - * When `true` indicates the entity has been soft deleted. - */ - deleted?: boolean; - /** - * Indicates if the workflow has been successfully deployed to Airflow. - */ - deployed?: boolean; - /** - * Description of the Pipeline. - */ - description?: string; - /** - * Display Name that identifies this Pipeline. - */ - displayName?: string; - /** - * Domains the asset belongs to. When not set, the asset inherits the domain from the parent - * it belongs to. - */ - domains?: EntityReference[]; - /** - * True if the pipeline is ready to be run in the next schedule. False if it is paused. - */ - enabled?: boolean; - /** - * Enable real-time log streaming to the OpenMetadata server. When enabled, ingestion logs - * will be automatically shipped to the server's configured log storage backend (S3 or - * compatible). - */ - enableStreamableLogs?: boolean; - /** - * Followers of this entity. - */ - followers?: EntityReference[]; - /** - * Name that uniquely identifies a Pipeline. - */ - fullyQualifiedName?: string; - /** - * Link to this ingestion pipeline resource. - */ - href?: string; - /** - * Unique identifier that identifies this pipeline. - */ - id?: string; - /** - * Bot user that performed the action on behalf of the actual user. - */ - impersonatedBy?: string; - /** - * Change that lead to this version of the entity. - */ - incrementalChangeDescription?: ChangeDescription; - /** - * The ingestion agent responsible for executing the ingestion pipeline. - */ - ingestionRunner?: EntityReference; - /** - * Set the logging level for the workflow. - */ - loggerLevel?: LogLevels; - /** - * Name that identifies this pipeline instance uniquely. - */ - name: string; - openMetadataServerConnection?: OpenMetadataConnection; - /** - * Owners of this Pipeline. - */ - owners?: EntityReference[]; - /** - * Last of executions and status for the Pipeline. - */ - pipelineStatuses?: PipelineStatus; - pipelineType: PipelineType; - /** - * The processing engine responsible for executing the ingestion pipeline logic. - */ - processingEngine?: EntityReference; - provider?: ProviderType; - /** - * Control if we want to flag the workflow as failed if we encounter any processing errors. - */ - raiseOnError?: boolean; - /** - * Link to the service (such as database, messaging, storage services, etc. for which this - * ingestion pipeline ingests the metadata from. - */ - service?: EntityReference; - sourceConfig: SourceConfig; - /** - * Last update time corresponding to the new version of the entity in Unix epoch time - * milliseconds. - */ - updatedAt?: number; - /** - * User who made the update. - */ - updatedBy?: string; - /** - * Metadata version of the entity. - */ - version?: number; -} - -/** - * Properties to configure the Airflow pipeline that will run the workflow. - */ -export interface AirflowConfig { - /** - * Concurrency of the Pipeline. - */ - concurrency?: number; - /** - * Email to notify workflow status. - */ - email?: string; - /** - * End Date of the pipeline. - */ - endDate?: Date; - /** - * Maximum Number of active runs. - */ - maxActiveRuns?: number; - /** - * pause the pipeline from running once the deploy is finished successfully. - */ - pausePipeline?: boolean; - /** - * Run past executions if the start date is in the past. - */ - pipelineCatchup?: boolean; - /** - * Timezone in which pipeline going to be scheduled. - */ - pipelineTimezone?: string; - /** - * Retry pipeline in case of failure. - */ - retries?: number; - /** - * Delay between retries in seconds. - */ - retryDelay?: number; - /** - * Scheduler Interval for the pipeline in cron format. - */ - scheduleInterval?: string; - /** - * Start date of the pipeline. - */ - startDate?: Date; - /** - * Default view in Airflow. - */ - workflowDefaultView?: string; - /** - * Default view Orientation in Airflow. - */ - workflowDefaultViewOrientation?: string; - /** - * Timeout for the workflow in seconds. - */ - workflowTimeout?: number; -} - -/** - * Change that led to this version of the entity. - * - * Description of the change. - * - * Change that lead to this version of the entity. - */ -export interface ChangeDescription { - changeSummary?: { [key: string]: ChangeSummary }; - /** - * Names of fields added during the version changes. - */ - fieldsAdded?: FieldChange[]; - /** - * Fields deleted during the version changes with old value before deleted. - */ - fieldsDeleted?: FieldChange[]; - /** - * Fields modified during the version changes with old and new values. - */ - fieldsUpdated?: FieldChange[]; - /** - * When a change did not result in change, this could be same as the current version. - */ - previousVersion?: number; -} - -export interface ChangeSummary { - changedAt?: number; - /** - * Name of the user or bot who made this change - */ - changedBy?: string; - changeSource?: ChangeSource; - [property: string]: any; -} - -/** - * The source of the change. This will change based on the context of the change (example: - * manual vs programmatic) - */ -export enum ChangeSource { - Automated = "Automated", - Derived = "Derived", - Ingested = "Ingested", - Manual = "Manual", - Propagated = "Propagated", - Suggested = "Suggested", -} - -export interface FieldChange { - /** - * Name of the entity field that changed. - */ - name?: string; - /** - * New value of the field. Note that this is a JSON string and use the corresponding field - * type to deserialize it. - */ - newValue?: any; - /** - * Previous value of the field. Note that this is a JSON string and use the corresponding - * field type to deserialize it. - */ - oldValue?: any; -} - -/** - * Domains the asset belongs to. When not set, the asset inherits the domain from the parent - * it belongs to. - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * The ingestion agent responsible for executing the ingestion pipeline. - * - * The processing engine responsible for executing the ingestion pipeline logic. - * - * Link to the service (such as database, messaging, storage services, etc. for which this - * ingestion pipeline ingests the metadata from. - * - * Service to be modified - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Set the logging level for the workflow. - * - * Supported logging levels - */ -export enum LogLevels { - Debug = "DEBUG", - Error = "ERROR", - Info = "INFO", - Warn = "WARN", -} - -/** - * OpenMetadata Connection Config - */ -export interface OpenMetadataConnection { - /** - * OpenMetadata server API version to use. - */ - apiVersion?: string; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: OpenMetadataServerConnectionElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - */ - hostPort: string; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: ConsumerConfigSSLClass; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: OpenmetadataType; - /** - * Flag to verify SSL Certificate for OpenMetadata Server. - */ - verifySSL?: VerifySSL; -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to only fetch databases that matches the pattern. - * - * Regex to only fetch tables or databases that matches the pattern. - * - * Regex to only fetch stored procedures that matches the pattern. - * - * Regex exclude tables or databases that matches the pattern. - * - * Regex exclude or include charts that matches the pattern. - * - * Regex to exclude or include dashboards that matches the pattern. - * - * Regex exclude or include data models that matches the pattern. - * - * Regex to exclude or include projects that matches the pattern. - * - * Regex to only fetch topics that matches the pattern. - * - * Regex to only compute metrics for table that matches the given tag, tiers, gloassary - * pattern. - * - * Regex exclude pipelines. - * - * Regex to only fetch MlModels with names matching the pattern. - * - * Regex to only fetch containers that matches the pattern. - * - * Regex to only include/exclude directories that matches the pattern. - * - * Regex to only include/exclude files that matches the pattern. - * - * Regex to only include/exclude spreadsheets that matches the pattern. - * - * Regex to only include/exclude worksheets that matches the pattern. - * - * Regex to only fetch search indexes that matches the pattern. - * - * Regex to only fetch api collections with names matching the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - * - * Regex to only include/exclude domains that match the pattern. - * - * Regex to only include/exclude glossaries that match the pattern. - * - * Regex to only fetch tags that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface OpenMetadataServerConnectionElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * SSL Configuration for OpenMetadata Server - * - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface ConsumerConfigSSLClass { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Service Type - * - * OpenMetadata service type - */ -export enum OpenmetadataType { - OpenMetadata = "OpenMetadata", -} - -/** - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * Last of executions and status for the Pipeline. - * - * This defines runtime status of Pipeline. - */ -export interface PipelineStatus { - /** - * Pipeline configuration for this particular execution. - */ - config?: { [key: string]: any }; - /** - * endDate of the pipeline run for this particular execution. - */ - endDate?: number; - /** - * Metadata for the pipeline status. - */ - metadata?: { [key: string]: any }; - /** - * Pipeline status denotes if its failed or succeeded. - */ - pipelineState?: PipelineState; - /** - * Pipeline unique run ID. - */ - runId?: string; - /** - * startDate of the pipeline run for this particular execution. - */ - startDate?: number; - /** - * Ingestion Pipeline summary status. Informed at the end of the execution. - */ - status?: StepSummary[]; - /** - * executionDate of the pipeline run for this particular execution. - */ - timestamp?: number; -} - -/** - * Pipeline status denotes if its failed or succeeded. - */ -export enum PipelineState { - Failed = "failed", - PartialSuccess = "partialSuccess", - Queued = "queued", - Running = "running", - Success = "success", -} - -/** - * Ingestion Pipeline summary status. Informed at the end of the execution. - * - * Summary for each step of the ingestion pipeline - * - * Defines the summary status of each step executed in an Ingestion Pipeline. - */ -export interface StepSummary { - /** - * Number of records with errors. - */ - errors?: number; - /** - * Sample of errors encountered in the step - */ - failures?: StackTraceError[]; - /** - * Number of filtered records. - */ - filtered?: number; - /** - * Step name - */ - name: string; - /** - * Detailed progress tracking by entity type (databases, schemas, tables, stored procedures) - */ - progress?: { [key: string]: Progress }; - /** - * Number of successfully processed records. - */ - records?: number; - /** - * Number of successfully updated records. - */ - updated_records?: number; - /** - * Number of records raising warnings. - */ - warnings?: number; -} - -/** - * Represents a failure status - */ -export interface StackTraceError { - /** - * Error being handled - */ - error: string; - /** - * Name of the asset with the error - */ - name: string; - /** - * Exception stack trace - */ - stackTrace?: string; -} - -export interface Progress { - /** - * Estimated remaining time in seconds for this entity type - */ - estimatedRemainingSeconds?: number; - /** - * Number of entities processed - */ - processed?: number; - /** - * Total number of entities discovered - */ - total?: number; - [property: string]: any; -} - -/** - * Type of Pipeline - metadata, usage - */ -export enum PipelineType { - Application = "application", - AutoClassification = "autoClassification", - DataInsight = "dataInsight", - Dbt = "dbt", - ElasticSearchReindex = "elasticSearchReindex", - Lineage = "lineage", - Metadata = "metadata", - Profiler = "profiler", - TestSuite = "TestSuite", - Usage = "usage", -} - -/** - * Type of provider of an entity. Some entities are provided by the `system`. Some are - * entities created and provided by the `user`. Typically `system` provide entities can't be - * deleted and can only be disabled. Some apps such as AutoPilot create entities with - * `automation` provider type. These entities can be deleted by the user. - */ -export enum ProviderType { - Automation = "automation", - System = "system", - User = "user", -} - -/** - * Additional connection configuration. - */ -export interface SourceConfig { - config?: Pipeline; -} - -/** - * DatabaseService Metadata Pipeline Configuration. - * - * DatabaseService Query Usage Pipeline Configuration. - * - * DatabaseService Query Lineage Pipeline Configuration. - * - * DashboardService Metadata Pipeline Configuration. - * - * MessagingService Metadata Pipeline Configuration. - * - * DatabaseService Profiler Pipeline Configuration. - * - * DatabaseService AutoClassification & Auto Classification Pipeline Configuration. - * - * PipelineService Metadata Pipeline Configuration. - * - * MlModelService Metadata Pipeline Configuration. - * - * StorageService Metadata Pipeline Configuration. - * - * DriveService Metadata Pipeline Configuration. - * - * SearchService Metadata Pipeline Configuration. - * - * TestSuite Pipeline Configuration. - * - * Data Insight Pipeline Configuration. - * - * DBT Pipeline Configuration. - * - * Application Pipeline Configuration. - * - * ApiService Metadata Pipeline Configuration. - * - * Apply a set of operations on a service - */ -export interface Pipeline { - /** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Optional configuration to toggle the DDL Statements ingestion. - */ - includeDDL?: boolean; - /** - * Set the 'Include Owners' toggle to control whether to include owners to the ingested - * entity if the owner email matches with a user stored in the OM server as part of metadata - * ingestion. If the ingested entity already exists and has an owner, the owner will not be - * overwritten. - * - * Enabling a flag will replace the current owner with a new owner from the source during - * metadata ingestion, if the current owner is null. It is recommended to keep the flag - * enabled to obtain the owner information during the first metadata ingestion. - */ - includeOwners?: boolean; - /** - * Optional configuration to toggle the Stored Procedures ingestion. - */ - includeStoredProcedures?: boolean; - /** - * Optional configuration to turn off fetching metadata for tables. - */ - includeTables?: boolean; - /** - * Optional configuration to toggle the tags ingestion. - */ - includeTags?: boolean; - /** - * Optional configuration to turn off fetching metadata for views. - */ - includeViews?: boolean; - /** - * Use incremental Metadata extraction after the first execution. This is commonly done by - * getting the changes from Audit tables on the supporting databases. - */ - incremental?: IncrementalMetadataExtractionConfiguration; - /** - * Optional configuration to soft delete databases in OpenMetadata if the source databases - * are deleted. Also, if the database is deleted, all the associated entities like schemas, - * tables, views, stored procedures, lineage, etc., with that database will be deleted - */ - markDeletedDatabases?: boolean; - /** - * Optional configuration to soft delete schemas in OpenMetadata if the source schemas are - * deleted. Also, if the schema is deleted, all the associated entities like tables, views, - * stored procedures, lineage, etc., with that schema will be deleted - */ - markDeletedSchemas?: boolean; - /** - * Optional configuration to soft delete stored procedures in OpenMetadata if the source - * stored procedures are deleted. Also, if the stored procedures is deleted, all the - * associated entities like lineage, etc., with that stored procedures will be deleted - */ - markDeletedStoredProcedures?: boolean; - /** - * This is an optional configuration for enabling soft deletion of tables. When this option - * is enabled, only tables that have been deleted from the source will be soft deleted, and - * this will apply solely to the schema that is currently being ingested via the pipeline. - * Any related entities such as test suites or lineage information that were associated with - * those tables will also be deleted. - */ - markDeletedTables?: boolean; - /** - * Set the 'Override Metadata' toggle to control whether to override the existing metadata - * in the OpenMetadata server with the metadata fetched from the source. If the toggle is - * set to true, the metadata fetched from the source will override the existing metadata in - * the OpenMetadata server. If the toggle is set to false, the metadata fetched from the - * source will not override the existing metadata in the OpenMetadata server. This is - * applicable for fields like description, tags, owner and displayName - */ - overrideMetadata?: boolean; - /** - * Advanced configuration for managing owners at multiple hierarchy levels (Service, - * Database, Schema, Table) with custom mappings and inheritance rules. - */ - ownerConfig?: OwnerConfiguration; - /** - * Configuration to tune how far we want to look back in query logs to process Stored - * Procedures results. - * - * Configuration to tune how far we want to look back in query logs to process usage data. - * - * Configuration to tune how far we want to look back in query logs to process lineage data. - */ - queryLogDuration?: number; - /** - * Configuration to set the timeout for parsing the query in seconds. - */ - queryParsingTimeoutLimit?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only fetch tables or databases that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex exclude tables or databases that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Number of Threads to use in order to parallelize Table ingestion. - * - * Number of Threads to use in order to parallelize lineage ingestion. - * - * Number of Threads to use in order to parallelize Drive ingestion. - */ - threads?: number; - /** - * Pipeline type - */ - type?: FluffyType; - /** - * Regex will be applied on fully qualified name (e.g - * service_name.db_name.schema_name.table_name) instead of raw name (e.g. table_name) - * - * Regex will be applied on fully qualified name (e.g service_name.directory_name.file_name) - * instead of raw name (e.g. file_name) - */ - useFqnForFiltering?: boolean; - /** - * Configuration the condition to filter the query history. - */ - filterCondition?: string; - /** - * Configuration to process query cost - */ - processQueryCostAnalysis?: boolean; - /** - * Configuration to set the file path for query logs - */ - queryLogFilePath?: string; - /** - * Configuration to set the limit for query logs - */ - resultLimit?: number; - /** - * Temporary file name to store the query logs before processing. Absolute file path - * required. - */ - stageFileLocation?: string; - /** - * Set 'Cross Database Service Names' to process lineage with the database. - */ - crossDatabaseServiceNames?: string[]; - /** - * Handle Lineage for Snowflake Temporary and Transient Tables. - */ - enableTempTableLineage?: boolean; - /** - * Set the 'Incremental Lineage Processing' toggle to control whether to process lineage - * incrementally. - */ - incrementalLineageProcessing?: boolean; - /** - * Set the 'Override View Lineage' toggle to control whether to override the existing view - * lineage. - */ - overrideViewLineage?: boolean; - /** - * Configuration to set the timeout for parsing the query in seconds. - */ - parsingTimeoutLimit?: number; - /** - * Set the 'Process Cross Database Lineage' toggle to control whether to process table - * lineage across different databases. - */ - processCrossDatabaseLineage?: boolean; - /** - * Set the 'Process Query Lineage' toggle to control whether to process query lineage. - */ - processQueryLineage?: boolean; - /** - * Set the 'Process Stored ProcedureLog Lineage' toggle to control whether to process stored - * procedure lineage. - */ - processStoredProcedureLineage?: boolean; - /** - * Set the 'Process View Lineage' toggle to control whether to process view lineage. - */ - processViewLineage?: boolean; - /** - * Regex to only fetch stored procedures that matches the pattern. - */ - storedProcedureFilterPattern?: FilterPattern; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Optional configuration to toggle the ingestion of data models. - */ - includeDataModels?: boolean; - /** - * Optional Configuration to include/exclude draft dashboards. By default it will include - * draft dashboards - */ - includeDraftDashboard?: boolean; - /** - * Details required to generate Lineage - */ - lineageInformation?: LineageInformation; - /** - * Optional configuration to soft delete dashboards in OpenMetadata if the source dashboards - * are deleted. Also, if the dashboard is deleted, all the associated entities like lineage, - * etc., with that dashboard will be deleted - */ - markDeletedDashboards?: boolean; - /** - * Optional configuration to soft delete data models in OpenMetadata if the source data - * models are deleted. Also, if the data models is deleted, all the associated entities like - * lineage, etc., with that data models will be deleted - */ - markDeletedDataModels?: boolean; - /** - * Set the 'Override Lineage' toggle to control whether to override the existing lineage. - */ - overrideLineage?: boolean; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * Option to turn on/off generating sample data during metadata extraction. - */ - generateSampleData?: boolean; - /** - * Optional configuration to soft delete topics in OpenMetadata if the source topics are - * deleted. Also, if the topic is deleted, all the associated entities like sample data, - * lineage, etc., with that topic will be deleted - */ - markDeletedTopics?: boolean; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Regex to only compute metrics for table that matches the given tag, tiers, gloassary - * pattern. - */ - classificationFilterPattern?: FilterPattern; - /** - * Option to turn on/off column metric computation. If enabled, profiler will compute column - * level metrics. - */ - computeColumnMetrics?: boolean; - /** - * Option to turn on/off computing profiler metrics. - */ - computeMetrics?: boolean; - /** - * Option to turn on/off table metric computation. If enabled, profiler will compute table - * level metrics. - */ - computeTableMetrics?: boolean; - processingEngine?: ProcessingEngine; - /** - * Percentage of data or no. of rows used to compute the profiler metrics and run data - * quality tests - * - * Percentage of data or no. of rows we want to execute the profiler and tests on - */ - profileSample?: number; - profileSampleType?: ProfileSampleType; - /** - * Whether to randomize the sample data or not. - */ - randomizedSample?: boolean; - samplingMethodType?: SamplingMethodType; - /** - * Number of threads to use during metric computations - */ - threadCount?: number; - /** - * Profiler Timeout in Seconds - */ - timeoutSeconds?: number; - /** - * Use system tables to extract metrics. Metrics that cannot be gathered from system tables - * will use the default methods. Using system tables can be faster but requires gathering - * statistics before running (for example using the ANALYZE procedure). More information can - * be found in the documentation: https://docs.openmetadata.org/latest/profler - */ - useStatistics?: boolean; - /** - * Set the Confidence value for which you want the column to be tagged as PII. Confidence - * value ranges from 0 to 100. A higher number will yield less false positives but more - * false negatives. A lower number will yield more false positives but less false negatives. - */ - confidence?: number; - /** - * Optional configuration to automatically tag columns that might contain sensitive - * information - */ - enableAutoClassification?: boolean; - /** - * Number of sample rows to ingest when 'Generate Sample Data' is enabled - */ - sampleDataCount?: number; - /** - * Option to turn on/off storing sample data. If enabled, we will ingest sample data for - * each table. - */ - storeSampleData?: boolean; - /** - * Optional configuration to turn off fetching lineage from pipelines. - */ - includeLineage?: boolean; - /** - * Optional configuration to toggle whether the un-deployed pipelines should be ingested or - * not. If set to false, only deployed pipelines will be ingested. - */ - includeUnDeployedPipelines?: boolean; - /** - * Optional configuration to soft delete Pipelines in OpenMetadata if the source Pipelines - * are deleted. Also, if the Pipeline is deleted, all the associated entities like lineage, - * etc., with that Pipeline will be deleted - */ - markDeletedPipelines?: boolean; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete MlModels in OpenMetadata if the source MlModels are - * deleted. Also, if the MlModel is deleted, all the associated entities like lineage, etc., - * with that MlModels will be deleted - */ - markDeletedMlModels?: boolean; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete containers in OpenMetadata if the source containers - * are deleted. Also, if the topic is deleted, all the associated entities with that - * containers will be deleted - */ - markDeletedContainers?: boolean; - storageMetadataConfigSource?: StorageMetadataConfigurationSource; - /** - * Regex to only include/exclude directories that matches the pattern. - */ - directoryFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude files that matches the pattern. - */ - fileFilterPattern?: FilterPattern; - /** - * Optional configuration to turn off fetching metadata for directories. - */ - includeDirectories?: boolean; - /** - * Optional configuration to turn off fetching metadata for files. - */ - includeFiles?: boolean; - /** - * Optional configuration to turn off fetching metadata for spreadsheets. - */ - includeSpreadsheets?: boolean; - /** - * Optional configuration to turn off fetching metadata for worksheets. - */ - includeWorksheets?: boolean; - /** - * Optional configuration to soft delete directories in OpenMetadata if the source - * directories are deleted. Also, if the directory is deleted, all the associated entities - * like files, spreadsheets, worksheets, lineage, etc., with that directory will be deleted - */ - markDeletedDirectories?: boolean; - /** - * Optional configuration to soft delete files in OpenMetadata if the source files are - * deleted. Also, if the file is deleted, all the associated entities like lineage, etc., - * with that file will be deleted - */ - markDeletedFiles?: boolean; - /** - * Optional configuration to soft delete spreadsheets in OpenMetadata if the source - * spreadsheets are deleted. Also, if the spreadsheet is deleted, all the associated - * entities like worksheets, lineage, etc., with that spreadsheet will be deleted - */ - markDeletedSpreadsheets?: boolean; - /** - * Optional configuration to soft delete worksheets in OpenMetadata if the source worksheets - * are deleted. Also, if the worksheet is deleted, all the associated entities like lineage, - * etc., with that worksheet will be deleted - */ - markDeletedWorksheets?: boolean; - /** - * Regex to only include/exclude spreadsheets that matches the pattern. - */ - spreadsheetFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude worksheets that matches the pattern. - */ - worksheetFilterPattern?: FilterPattern; - /** - * Enable the 'Include Index Template' toggle to manage the ingestion of index template data. - */ - includeIndexTemplate?: boolean; - /** - * Optional configuration to turn off fetching sample data for search index. - */ - includeSampleData?: boolean; - /** - * Optional configuration to soft delete search indexes in OpenMetadata if the source search - * indexes are deleted. Also, if the search index is deleted, all the associated entities - * like lineage, etc., with that search index will be deleted - */ - markDeletedSearchIndexes?: boolean; - /** - * No. of records of sample data we want to ingest. - */ - sampleSize?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Fully qualified name of the entity to be tested, if we're working with a basic suite. - */ - entityFullyQualifiedName?: string; - /** - * Service connections to be used for the logical test suite. - */ - serviceConnections?: ServiceConnections[]; - /** - * List of test cases to be executed on the entity. If null, all test cases will be executed. - */ - testCases?: string[]; - /** - * Maximum number of events entities in a batch (Default 1000). - */ - batchSize?: number; - /** - * Certificate path to be added in configuration. The path should be local in the Ingestion - * Container. - */ - caCerts?: string; - recreateIndex?: boolean; - /** - * Region name. Required when using AWS Credentials. - */ - regionName?: string; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Connection Timeout - */ - timeout?: number; - /** - * Indicates whether to use aws credentials when connecting to OpenSearch in AWS. - */ - useAwsCredentials?: boolean; - /** - * Indicates whether to use SSL when connecting to ElasticSearch. By default, we will ignore - * SSL settings. - */ - useSSL?: boolean; - /** - * Indicates whether to verify certificates when using SSL connection to ElasticSearch. - * Ignored by default. Is set to true, make sure to send the certificates in the property - * `CA Certificates`. - */ - verifyCerts?: boolean; - /** - * Custom OpenMetadata Classification name for dbt tags. - */ - dbtClassificationName?: string; - /** - * Available sources to fetch DBT catalog and manifest files. - */ - dbtConfigSource?: DBTConfigurationSource; - /** - * Optional configuration to update the description from DBT or not - */ - dbtUpdateDescriptions?: boolean; - /** - * Optional configuration to update the owners from DBT or not - */ - dbtUpdateOwners?: boolean; - /** - * Optional configuration to search across databases for tables or not - */ - searchAcrossDatabases?: boolean; - /** - * Regex to only fetch tags that matches the pattern. - */ - tagFilterPattern?: FilterPattern; - /** - * Application configuration - */ - appConfig?: any[] | boolean | number | null | CollateAIAppConfig | string; - /** - * Application private configuration - */ - appPrivateConfig?: PrivateConfig; - /** - * Source Python Class Name to run the application - */ - sourcePythonClass?: string; - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete api collections in OpenMetadata if the source - * collections are deleted. Also, if the collection is deleted, all the associated entities - * like endpoints, etc., with that collection will be deleted - */ - markDeletedApiCollections?: boolean; - /** - * Optional value of the ingestion runner name responsible for running the workflow - */ - ingestionRunner?: string; - /** - * List of operations to be performed on the service - */ - operations?: Operation[]; - /** - * Service to be modified - */ - service?: EntityReference; -} - -/** - * Configuration for the CollateAI External Application. - * - * Configuration for the Automator External Application. - * - * This schema defines the Slack App Token Configuration - * - * Configuration for the Collate AI Quality Agent. - * - * No configuration needed to instantiate the Data Insights Pipeline. The logic is handled - * in the backend. - * - * Search Indexing App. - * - * Cache Warmup Application Configuration. - * - * Configuration for the AutoPilot Application. - */ -export interface CollateAIAppConfig { - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - filter?: string; - /** - * Patch the description if it is empty, instead of raising a suggestion - * - * Patch the tier if it is empty, instead of raising a suggestion - */ - patchIfEmpty?: boolean; - /** - * Application Type - */ - type?: CollateAIAppConfigType; - /** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - */ - actions?: Action[]; - /** - * Entities selected to run the automation. - */ - resources?: Resource; - /** - * Bot Token - */ - botToken?: string; - /** - * User Token - */ - userToken?: string; - /** - * Whether the suggested tests should be active or not upon suggestion - * - * Whether the AutoPilot Workflow should be active or not. - */ - active?: boolean; - backfillConfiguration?: BackfillConfiguration; - /** - * Maximum number of events processed at a time (Default 100). - * - * Maximum number of events sent in a batch (Default 100). - * - * Number of entities to process in each batch. - */ - batchSize?: number; - moduleConfiguration?: ModuleConfiguration; - /** - * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property - * Type and are facing errors. Bear in mind that recreating the index will delete your - * DataAssets and a backfill will be needed. - */ - recreateDataAssetsIndex?: boolean; - sendToAdmins?: boolean; - sendToTeams?: boolean; - /** - * Enable automatic performance tuning based on cluster capabilities and database entity - * count - */ - autoTune?: boolean; - /** - * Number of threads to use for reindexing - * - * Number of parallel threads for processing entities and warming cache. - */ - consumerThreads?: number; - /** - * List of Entities to Reindex - * - * List of entity types to warm up in cache. Use 'all' to warm up all entity types. - */ - entities?: string[]; - /** - * Initial backoff time in milliseconds - */ - initialBackoff?: number; - /** - * Maximum backoff time in milliseconds - */ - maxBackoff?: number; - /** - * Maximum number of concurrent requests to the search index - */ - maxConcurrentRequests?: number; - /** - * Maximum number of retries for a failed request - */ - maxRetries?: number; - /** - * Maximum number of events sent in a batch (Default 100). - */ - payLoadSize?: number; - /** - * Number of threads to use for reindexing - */ - producerThreads?: number; - /** - * Queue Size to user internally for reindexing. - * - * Internal queue size for entity processing pipeline. - */ - queueSize?: number; - /** - * This schema publisher run modes. - */ - recreateIndex?: boolean; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Force cache warmup even if another instance is detected (use with caution). - */ - force?: boolean; - /** - * Enter the retention period for Activity Threads of type = 'Conversation' records in days - * (e.g., 30 for one month, 60 for two months). - */ - activityThreadsRetentionPeriod?: number; - /** - * Enter the retention period for change event records in days (e.g., 7 for one week, 30 for - * one month). - */ - changeEventRetentionPeriod?: number; - /** - * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two - * months). - */ - profileDataRetentionPeriod?: number; - /** - * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for - * two months). - */ - testCaseResultsRetentionPeriod?: number; - /** - * Service Entity Link for which to trigger the application. - */ - entityLink?: string; - [property: string]: any; -} - -/** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - * - * Apply Classification Tags to the selected assets. - * - * Remove Classification Tags Action Type - * - * Apply Glossary Terms to the selected assets. - * - * Remove Glossary Terms Action Type - * - * Add domains to the selected assets. - * - * Remove domains from the selected assets. - * - * Apply Tags to the selected assets. - * - * Add a Custom Property to the selected assets. - * - * Remove Owner Action Type - * - * Add an owner to the selected assets. - * - * Add Test Cases to the selected assets. - * - * Remove Test Cases Action Type - * - * Add owners to the selected assets. - * - * Remove Custom Properties Action Type - * - * Add a Data Product to the selected assets. - * - * Remove a Data Product to the selected assets. - * - * Propagate description, tags and glossary terms via lineage - * - * ML Tagging action configuration for external automator. - */ -export interface Action { - /** - * Apply tags to the children of the selected assets that match the criteria. E.g., columns, - * tasks, topic fields,... - * - * Remove tags from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply terms to the children of the selected assets that match the criteria. E.g., - * columns, tasks, topic fields,... - * - * Remove terms from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply the description to the children of the selected assets that match the criteria. - * E.g., columns, tasks, topic fields,... - * - * Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Add tests to the selected table columns - * - * Remove tests to the selected table columns - */ - applyToChildren?: string[]; - /** - * Update tags even if they are already defined in the asset. By default, incoming tags are - * merged with the existing ones. - * - * Update terms even if they are already defined in the asset. By default, incoming terms - * are merged with the existing ones. - * - * Update the domains even if they are defined in the asset. By default, we will only apply - * the domains to assets without domains. - * - * Update the description even if they are already defined in the asset. By default, we'll - * only add the descriptions to assets without the description set. - * - * Update the Custom Property even if it is defined in the asset. By default, we will only - * apply the owners to assets without the given Custom Property informed. - * - * Update the tier even if it is defined in the asset. By default, we will only apply the - * tier to assets without tier. - * - * Update the test even if it is defined in the asset. By default, we will only apply the - * test to assets without the existing test already existing. - * - * Update the owners even if it is defined in the asset. By default, we will only apply the - * owners to assets without owner. - * - * Update the Data Product even if the asset belongs to a different Domain. By default, we - * will only add the Data Product if the asset has no Domain, or it belongs to the same - * domain as the Data Product. - * - * Update descriptions, tags and Glossary Terms via lineage even if they are already defined - * in the asset. By default, descriptions are only updated if they are not already defined - * in the asset, and incoming tags are merged with the existing ones. - */ - overwriteMetadata?: boolean; - /** - * Classification Tags to apply (source must be 'Classification') - * - * Classification Tags to remove (source must be 'Classification') - */ - tags?: TierElement[]; - /** - * Application Type - */ - type: ActionType; - /** - * Remove tags from all the children and parent of the selected assets. - * - * Remove terms from all the children and parent of the selected assets. - * - * Remove descriptions from all the children and parent of the selected assets. - */ - applyToAll?: boolean; - /** - * Remove tags by its label type - * - * Remove terms by its label type - */ - labels?: LabelElement[]; - /** - * Glossary Terms to apply - * - * Glossary Terms to remove - */ - terms?: TierElement[]; - /** - * Domains to apply - */ - domains?: EntityReference[]; - /** - * Description to apply - */ - description?: string; - /** - * Owners to apply - * - * Custom Properties keys to remove - */ - customProperties?: any; - /** - * tier to apply - */ - tier?: TierElement; - /** - * Test Cases to apply - */ - testCases?: TestCaseDefinitions[]; - /** - * Remove all test cases - */ - removeAll?: boolean; - /** - * Test Cases to remove - */ - testCaseDefinitions?: string[]; - /** - * Owners to apply - */ - owners?: EntityReference[]; - /** - * Data Products to apply - * - * Data Products to remove - */ - dataProducts?: EntityReference[]; - /** - * Propagate the metadata to columns via column-level lineage. - */ - propagateColumnLevel?: boolean; - /** - * Propagate description through lineage - */ - propagateDescription?: boolean; - /** - * Propagate domains from the parent through lineage - */ - propagateDomains?: boolean; - /** - * Propagate glossary terms through lineage - */ - propagateGlossaryTerms?: boolean; - /** - * Propagate owner from the parent - */ - propagateOwner?: boolean; - /** - * Propagate the metadata to the parents (e.g., tables) via lineage. - */ - propagateParent?: boolean; - /** - * Propagate tags through lineage - */ - propagateTags?: boolean; - /** - * Propagate tier from the parent - */ - propagateTier?: boolean; - /** - * Number of levels to propagate lineage. If not set, it will propagate to all levels. - */ - propagationDepth?: number; - /** - * List of configurations to stop propagation based on conditions - */ - propagationStopConfigs?: PropagationStopConfig[]; -} - -/** - * Remove tags by its label type - * - * Remove terms by its label type - */ -export enum LabelElement { - Automated = "Automated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Configuration to stop lineage propagation based on conditions - */ -export interface PropagationStopConfig { - /** - * The metadata attribute to check for stopping propagation - */ - metadataAttribute: MetadataAttribute; - /** - * List of attribute values that will stop propagation when any of them is matched - */ - value: Array; -} - -/** - * The metadata attribute to check for stopping propagation - */ -export enum MetadataAttribute { - Description = "description", - Domains = "domains", - GlossaryTerms = "glossaryTerms", - Owner = "owner", - Tags = "tags", - Tier = "tier", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - * - * Domains the asset belongs to. When not set, the asset inherits the domain from the parent - * it belongs to. - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * The ingestion agent responsible for executing the ingestion pipeline. - * - * The processing engine responsible for executing the ingestion pipeline logic. - * - * Link to the service (such as database, messaging, storage services, etc. for which this - * ingestion pipeline ingests the metadata from. - * - * Service to be modified - */ -export interface TagLabel { - /** - * Description for the tag label. - * - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this tag. - * - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Link to the tag resource. - * - * Link to the entity resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType?: LabelTypeEnum; - /** - * Name of the tag or glossary term. - * - * Name of the entity instance. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source?: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state?: State; - style?: Style; - tagFQN?: string; - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id?: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type?: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelTypeEnum { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * Cover image configuration for the entity. - */ - coverImage?: CoverImage; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * Cover image configuration for the entity. - * - * Cover image configuration for an entity. This is used to display a banner or header image - * for entities like Domain, Glossary, Data Product, etc. - */ -export interface CoverImage { - /** - * Position of the cover image in CSS background-position format. Supports keywords (top, - * center, bottom) or pixel values (e.g., '20px 30px'). - */ - position?: string; - /** - * URL of the cover image. - */ - url?: string; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - */ -export interface TierElement { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelTypeEnum; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Minimum set of requirements to get a Test Case request ready - */ -export interface TestCaseDefinitions { - /** - * Compute the passed and failed row count for the test case. - */ - computePassedFailedRowCount?: boolean; - parameterValues?: TestCaseParameterValue[]; - /** - * Tags to apply - */ - tags?: TierElement[]; - /** - * Fully qualified name of the test definition. - */ - testDefinition?: string; - /** - * If the test definition supports it, use dynamic assertion to evaluate the test case. - */ - useDynamicAssertion?: boolean; - [property: string]: any; -} - -/** - * This schema defines the parameter values that can be passed for a Test Case. - */ -export interface TestCaseParameterValue { - /** - * name of the parameter. Must match the parameter names in testCaseParameterDefinition - */ - name?: string; - /** - * value to be passed for the Parameters. These are input from Users. We capture this in - * string and convert during the runtime. - */ - value?: string; - [property: string]: any; -} - -/** - * Application Type - * - * Add Tags action type. - * - * Remove Classification Tags Action Type. - * - * Add Terms action type. - * - * Remove Terms Action Type. - * - * Add Domain Action Type. - * - * Remove Domain Action Type - * - * Add Description Action Type. - * - * Add Custom Properties Action Type. - * - * Remove Description Action Type - * - * Add Tier Action Type. - * - * Remove Tier Action Type - * - * Add Test Case Action Type. - * - * Remove Test Case Action Type - * - * Add Owner Action Type. - * - * Remove Owner Action Type - * - * Remove Custom Properties Action Type. - * - * Add Data Products Action Type. - * - * Remove Data Products Action Type. - * - * Lineage propagation action type. - * - * ML PII Tagging action type. - */ -export enum ActionType { - AddCustomPropertiesAction = "AddCustomPropertiesAction", - AddDataProductAction = "AddDataProductAction", - AddDescriptionAction = "AddDescriptionAction", - AddDomainAction = "AddDomainAction", - AddOwnerAction = "AddOwnerAction", - AddTagsAction = "AddTagsAction", - AddTermsAction = "AddTermsAction", - AddTestCaseAction = "AddTestCaseAction", - AddTierAction = "AddTierAction", - LineagePropagationAction = "LineagePropagationAction", - MLTaggingAction = "MLTaggingAction", - RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", - RemoveDataProductAction = "RemoveDataProductAction", - RemoveDescriptionAction = "RemoveDescriptionAction", - RemoveDomainAction = "RemoveDomainAction", - RemoveOwnerAction = "RemoveOwnerAction", - RemoveTagsAction = "RemoveTagsAction", - RemoveTermsAction = "RemoveTermsAction", - RemoveTestCaseAction = "RemoveTestCaseAction", - RemoveTierAction = "RemoveTierAction", -} - -/** - * Backfill Configuration - */ -export interface BackfillConfiguration { - /** - * Enable Backfill for the configured dates - */ - enabled?: boolean; - /** - * Date for which the backfill will end - */ - endDate?: Date; - /** - * Date from which to start the backfill - */ - startDate?: Date; - [property: string]: any; -} - -/** - * Different Module Configurations - */ -export interface ModuleConfiguration { - /** - * App Analytics Module configuration - */ - appAnalytics: AppAnalyticsConfig; - /** - * Cost Analysis Insights Module configuration - */ - costAnalysis: CostAnalysisConfig; - /** - * Data Assets Insights Module configuration - */ - dataAssets: DataAssetsConfig; - /** - * Data Quality Insights Module configuration - */ - dataQuality: DataQualityConfig; -} - -/** - * App Analytics Module configuration - */ -export interface AppAnalyticsConfig { - /** - * If Enabled, App Analytics insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Cost Analysis Insights Module configuration - */ -export interface CostAnalysisConfig { - /** - * If Enabled, Cost Analysis insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Data Assets Insights Module configuration - */ -export interface DataAssetsConfig { - /** - * If Enabled, Data Asset insights will be populated when the App runs. - */ - enabled: boolean; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Defines the number of days the Data Assets Insights information will be kept. After it - * they will be deleted. - */ - retention?: number; - serviceFilter?: ServiceFilter; -} - -export interface ServiceFilter { - serviceName?: string; - serviceType?: string; -} - -/** - * Data Quality Insights Module configuration - */ -export interface DataQualityConfig { - /** - * If Enabled, Data Quality insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Entities selected to run the automation. - */ -export interface Resource { - /** - * Filter JSON tree to be used for rendering the filters in the UI. This comes from - * Immutable Tree type of react-awesome-query-builder. - */ - filterJsonTree?: string; - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - queryFilter?: string; - /** - * Type of the entity. E.g., 'table', 'chart',... - */ - type?: string[]; - [property: string]: any; -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Application Type - * - * Application type. - */ -export enum CollateAIAppConfigType { - AutoPilotApplication = "AutoPilotApplication", - Automator = "Automator", - CacheWarmup = "CacheWarmup", - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - CollateAITierAgent = "CollateAITierAgent", - DataInsights = "DataInsights", - DataInsightsReport = "DataInsightsReport", - SearchIndexing = "SearchIndexing", -} - -/** - * Application private configuration - * - * Private Configuration for the CollateAI External Application. - */ -export interface PrivateConfig { - /** - * Collate Server public URL. WAII will use this information to interact with the server. - * E.g., https://sandbox.getcollate.io - */ - collateURL?: string; - /** - * Limits for the CollateAI Application. - */ - limits?: AppLimitsConfig; - /** - * WAII API Token - */ - token?: string; - /** - * WAII API host URL - */ - waiiInstance?: string; - [property: string]: any; -} - -/** - * Limits for the CollateAI Application. - * - * Private Configuration for the App Limits. - */ -export interface AppLimitsConfig { - /** - * The records of the limits. - */ - actions: { [key: string]: number }; - /** - * The start of this limit cycle. - */ - billingCycleStart: Date; -} - -/** - * Available sources to fetch DBT catalog and manifest files. - * - * dbt Cloud configuration. - * - * DBT Catalog, Manifest and Run Results file path config. - * - * DBT Catalog, Manifest and Run Results HTTP path configuration. - * - * DBT Catalog, Manifest and Run Results files in S3 bucket. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in GCS storage. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in Azure bucket. We will search for - * catalog.json, manifest.json and run_results.json. - */ -export interface DBTConfigurationSource { - /** - * dbt cloud account Id - */ - dbtCloudAccountId?: string; - /** - * dbt cloud account authentication token - */ - dbtCloudAuthToken?: string; - /** - * dbt cloud job id. - */ - dbtCloudJobId?: string; - /** - * In case of multiple projects in a dbt cloud account, specify the project's id from which - * you want to extract the dbt run artifacts - */ - dbtCloudProjectId?: string; - /** - * URL to connect to your dbt cloud instance. E.g., https://cloud.getdbt.com or - * https://emea.dbt.com/ - */ - dbtCloudUrl?: string; - /** - * dbt Configuration type - */ - dbtConfigType: DbtConfigType; - /** - * DBT catalog file path to extract dbt models with their column schemas. - */ - dbtCatalogFilePath?: string; - /** - * DBT manifest file path to extract dbt models and associate with tables. - */ - dbtManifestFilePath?: string; - /** - * DBT run results file path to extract the test results information. - */ - dbtRunResultsFilePath?: string; - /** - * DBT sources file path to extract the freshness test result. - */ - dbtSourcesFilePath?: string; - /** - * DBT catalog http file path to extract dbt models with their column schemas. - */ - dbtCatalogHttpPath?: string; - /** - * DBT manifest http file path to extract dbt models and associate with tables. - */ - dbtManifestHttpPath?: string; - /** - * DBT run results http file path to extract the test results information. - */ - dbtRunResultsHttpPath?: string; - /** - * DBT sources http file path to extract freshness test results information. - */ - dbtSourcesHttpPath?: string; - /** - * Details of the bucket where the dbt files are stored - */ - dbtPrefixConfig?: DBTPrefixConfig; - dbtSecurityConfig?: Credentials; -} - -/** - * dbt Configuration type - */ -export enum DbtConfigType { - Azure = "azure", - Cloud = "cloud", - Gcs = "gcs", - HTTP = "http", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the dbt files are stored - */ -export interface DBTPrefixConfig { - /** - * Name of the bucket where the dbt files are stored - */ - dbtBucketName?: string; - /** - * Path of the folder where the dbt files are stored - */ - dbtObjectPrefix?: string; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - * - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Use incremental Metadata extraction after the first execution. This is commonly done by - * getting the changes from Audit tables on the supporting databases. - */ -export interface IncrementalMetadataExtractionConfiguration { - /** - * If True, enables Metadata Extraction to be incremental - */ - enabled: boolean; - /** - * Number os days to search back for a successful pipeline run. The timestamp of the last - * found successful pipeline run will be used as a base to search for updated entities. - */ - lookbackDays?: number; - /** - * Number of days to add to the last successful pipeline run timestamp to search for updated - * entities. - */ - safetyMarginDays?: number; -} - -/** - * Details required to generate Lineage - */ -export interface LineageInformation { - /** - * List of service path prefixes for lineage matching. Supported formats: DBServiceName, - * DBServiceName.DatabaseName, DBServiceName.DatabaseName.SchemaName, or - * DBServiceName.DatabaseName.SchemaName.TableName - */ - dbServicePrefixes?: string[]; - /** - * List of Database Service Names for creation of lineage - */ - dbServiceNames?: string[]; - /** - * List of Storage Service Names for creation of lineage - */ - storageServiceNames?: string[]; - [property: string]: any; -} - -/** - * Operation to be performed on the entity - */ -export interface Operation { - /** - * Entity to be modified - */ - entityLink: string; - /** - * The id of the operation - */ - id: string; - /** - * The configuration for the operation to be applied - */ - parameters: ReverseIngestionConfig; - /** - * Templated SQL command to be used for the operation. Context parameters will be populated - * based on the event type. - */ - SQLTemplate?: string; - /** - * Type of operation to perform - */ - type: OperationType; -} - -/** - * The configuration for the operation to be applied - * - * Configuration for updating descriptions - * - * Configuration for updating owners - * - * Configuration for updating tags - */ -export interface ReverseIngestionConfig { - /** - * New description of the service - */ - newDescription?: string; - /** - * Previous description of the service - */ - previousDescription?: string; - /** - * Added owners to be applied - */ - addedOwners?: EntityReference[]; - /** - * Removed owners from the entity - */ - removedOwners?: EntityReference[]; - /** - * Added tags to be applied - */ - addedTags?: TierElement[]; - /** - * Removed tags of the entity - */ - removedTags?: TierElement[]; -} - -/** - * Type of operation to perform - */ -export enum OperationType { - UpdateDescription = "UPDATE_DESCRIPTION", - UpdateOwner = "UPDATE_OWNER", - UpdateTags = "UPDATE_TAGS", -} - -/** - * Advanced configuration for managing owners at multiple hierarchy levels (Service, - * Database, Schema, Table) with custom mappings and inheritance rules. - * - * Configuration for assigning owners to ingested entities following topology hierarchy with - * inheritance support - */ -export interface OwnerConfiguration { - /** - * Owner for database entities. Can be a single owner for all databases, or a map of - * database names to owner(s). - */ - database?: { [key: string]: string[] | string } | string; - /** - * Owner for schema entities. Can be a single owner for all schemas, or a map of schema FQNs - * to owner(s). - */ - databaseSchema?: { [key: string]: string[] | string } | string; - /** - * Default owner applied to all entities when no specific owner is configured (user or team - * name/email) - */ - default?: string; - /** - * Enable child entities to inherit owner from parent entities when they don't have a - * specific owner configured - */ - enableInheritance?: boolean; - /** - * Owner for the service level - */ - service?: string; - /** - * Owner for table entities. Can be a single owner for all tables, or a map of table FQNs to - * owner(s). - */ - table?: { [key: string]: string[] | string } | string; -} - -/** - * Processing Engine Configuration. If not provided, the Native Engine will be used by - * default. - * - * Configuration for the native metadata ingestion engine - * - * This schema defines the configuration for a Spark Engine runner. - */ -export interface ProcessingEngine { - /** - * The type of the engine configuration - */ - type: ProcessingEngineType; - config?: Config; - /** - * Spark Connect Remote URL. - */ - remote?: string; -} - -export interface Config { - /** - * Additional Spark configuration properties as key-value pairs. - */ - extraConfig?: { [key: string]: any }; - /** - * Temporary path to store the data. - */ - tempPath?: string; - [property: string]: any; -} - -/** - * The type of the engine configuration - */ -export enum ProcessingEngineType { - Native = "Native", - Spark = "Spark", -} - -/** - * Type of Profile Sample (percentage or rows) - */ -export enum ProfileSampleType { - Percentage = "PERCENTAGE", - Rows = "ROWS", -} - -/** - * Type of Sampling Method (BERNOULLI or SYSTEM) - */ -export enum SamplingMethodType { - Bernoulli = "BERNOULLI", - System = "SYSTEM", -} - -/** - * Service connections available for the logical test suite. - */ -export interface ServiceConnections { - /** - * Connection configuration for the source. ex: mysql , tableau connection. - */ - serviceConnection: ServiceConnection; - serviceName: string; -} - -/** - * Connection configuration for the source. ex: mysql , tableau connection. - * - * Supported services - * - * API Service Connection. - * - * Dashboard Connection. - * - * Database Connection. - * - * Metadata Service Connection. - * - * Pipeline Connection. - * - * MlModel Connection. - * - * Storage Connection. - * - * search Connection. - * - * Security Connection. - * - * Drive Connection. - */ -export interface ServiceConnection { - config?: ConfigObject; -} - -/** - * REST Connection Config - * - * Looker Connection Config - * - * Metabase Connection Config - * - * PowerBI Connection Config - * - * PowerBIReportServer Connection Config - * - * Redash Connection Config - * - * Superset Connection Config - * - * Tableau Connection Config - * - * Mode Connection Config - * - * Custom Dashboard Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Domo Dashboard Connection Config - * - * QuickSight Connection Config - * - * Qlik Sense Connection Config - * - * Lightdash Connection Config - * - * MicroStrategy Connection Config - * - * Qlik Cloud Connection Config - * - * Sigma Connection Config - * - * ThoughtSpot Connection Config - * - * Grafana Connection Config - * - * Hex Connection Config - * - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * TimescaleDB Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - * - * Kafka Connection Config - * - * Redpanda Connection Config - * - * Kinesis Connection Config - * - * Custom Messaging Service Connection to build a source that is not supported by - * OpenMetadata yet. - * - * Amundsen Connection Config - * - * Metadata to ElasticSearch Connection Config - * - * OpenMetadata Connection Config - * - * Atlas Connection Config - * - * Alation Connection Config - * - * Alation Sink Connection Config - * - * Collibra Connection Config - * - * Airflow Metadata Database Connection Config - * - * Wherescape Metadata Database Connection Config - * - * SSIS Metadata Database Connection Config - * - * Glue Pipeline Connection Config - * - * AWS Kinesis Firehose Pipeline Connection Config - * - * Airbyte Metadata Database Connection Config - * - * Fivetran Metadata Database Connection Config - * - * Flink Metadata Connection Config - * - * Dagster Metadata Database Connection Config - * - * Nifi Metadata Pipeline Connection Config - * - * Domo Pipeline Connection Config - * - * Custom Pipeline Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Spline Metadata Database Connection Config - * - * Spark Metadata Pipeline Connection Config - * - * OpenLineage Connection Config - * - * KafkaConnect Connection Config - * - * DBTCloud Connection Config - * - * Matillion Connection - * - * Azure Data Factory Connection Config - * - * Stitch Connection - * - * Snowplow Pipeline Connection Config - * - * MlFlow Connection Config - * - * Sklearn Connection Config - * - * Custom MlModel Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * SageMaker Connection Config - * - * Google VertexAI Connection Config - * - * S3 Connection. - * - * ADLS Connection. - * - * GCS Connection. - * - * Custom Storage Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * ElasticSearch Connection. - * - * OpenSearch Connection Config - * - * Custom Search Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * Apache Ranger Connection Config - * - * Google Drive Connection Config - * - * SharePoint Connection Config - * - * Custom Drive Connection to build a source that is not supported. - */ -export interface ConfigObject { - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Documentation URL for the schema. - */ - docURL?: string; - /** - * Open API Schema URL. - */ - openAPISchemaURL?: string; - /** - * Supports Metadata Extraction. - */ - supportsMetadataExtraction?: boolean; - /** - * Generated Token to connect to OpenAPI Schema. - * - * token to connect to Qlik Cloud. - * - * Hex API token for authentication. Can be personal or workspace token. - * - * To Connect to Dagster Cloud - * - * Generated Token to connect to Databricks. - * - * Generated Token to connect to DBTCloud. - * - * Token to connect to Stitch api doc - */ - token?: string; - /** - * REST API Type - * - * Service Type - * - * Custom dashboard service type - * - * Custom database service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * OpenSearch Type - * - * Custom search service type - */ - type?: PurpleType; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * User's Client ID. This user should have privileges to read all the metadata in Looker. - * - * client_id for PowerBI. - * - * Client ID for DOMO - * - * client_id for Sigma. - * - * Azure Application (client) ID for service principal authentication. - * - * Application (client) ID from Azure Active Directory - */ - clientId?: string; - /** - * User's Client Secret. - * - * clientSecret for PowerBI. - * - * clientSecret for Sigma. - * - * Azure Application client secret for service principal authentication. - * - * Application (client) secret from Azure Active Directory - */ - clientSecret?: string; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - */ - gitCredentials?: NoGitCredentialsClass | string; - /** - * URL to the Looker instance. - * - * Host and Port of the Metabase instance. - * - * Dashboard URL for PowerBI service. - * - * Dashboard URL for PowerBI Report Server. - * - * URL for the Redash instance - * - * URL for the superset instance. - * - * Tableau Server url. - * - * URL for the mode instance. - * - * URL for the Qlik instance. - * - * Address for your running Lightdash instance - * - * Host and Port of the MicroStrategy instance. - * - * Host and Port of the Qlik Cloud instance. - * - * Sigma API url. - * - * ThoughtSpot instance URL. Example: https://my-company.thoughtspot.cloud - * - * URL to the Grafana instance. - * - * Hex API URL. For Hex.tech cloud, use https://app.hex.tech - * - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the TimescaleDB service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - * - * Host and port of the Amundsen Neo4j Connection. This expect a URI format like: - * bolt://localhost:7687. - * - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - * - * Host and port of the Atlas service. - * - * Host and port of the Alation service. - * - * Host and port of the Collibra service. - * - * Pipeline Service Management/UI URI. - * - * Pipeline Service Management/UI URL. - * - * Spline REST Server Host & Port. - * - * KafkaConnect Service Management/UI URI. - * - * Host and port of the Stitch API host - * - * Host and port of the ElasticSearch service. - * - * Host and port of the OpenSearch service. - * - * Apache Ranger Admin URL. - */ - hostPort?: string; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * API token to connect to Metabase. Use this instead of username/password for token-based - * authentication. - * - * API key of the redash instance to access. - * - * The personal access token you can generate in the Lightdash app under the user settings - * - * Service Account Token to authenticate to the Grafana APIs. Use Service Account Tokens - * (format: glsa_xxxx) for authentication. Legacy API Keys are no longer supported by - * Grafana as of January 2025. Both self-hosted and Grafana Cloud are supported. Requires - * Admin role for full metadata extraction. - * - * API key to authenticate with the SAP ERP APIs. - * - * Fivetran API Key. - * - * API Key for Snowplow Console API - */ - apiKey?: string; - /** - * Password to connect to Metabase. Required for basic authentication. - * - * Password to connect to PowerBI report server. - * - * Password to connect to MicroStrategy. - * - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - * - * password to connect to the Amundsen Neo4j Connection. - * - * password to connect to the Atlas. - * - * Password to connect to the Collibra. - */ - password?: string; - /** - * Username to connect to Metabase. Required for basic authentication. - * - * Username to connect to PowerBI report server. - * - * Username for Redash - * - * Username to connect to MicroStrategy. This user should have privileges to read all the - * metadata in MicroStrategy. - * - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to TimescaleDB. This user should have privileges to read all the - * metadata in TimescaleDB. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to Salesforce. This user should have privileges to read all the - * metadata in Salesforce. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - * - * username to connect to the Amundsen Neo4j Connection. - * - * username to connect to the Atlas. This user should have privileges to read all the - * metadata in Atlas. - * - * Username to connect to the Collibra. This user should have privileges to read all the - * metadata in Collibra. - */ - username?: string; - /** - * API URL to call powerbi rest apis to extract metadata. Default to - * `https://api.powerbi.com`. You can provide youw own in case of different environment - */ - apiURL?: string; - /** - * Authority URI for the PowerBI service. - */ - authorityURI?: string; - /** - * Display Table Name from source instead of renamed table name for datamodel tables - */ - displayTableNameFromSource?: boolean; - /** - * Entity Limit set here will be used to paginate the PowerBi APIs - */ - pagination_entity_per_page?: number; - /** - * Source to get the .pbit files to extract lineage information - */ - pbitFilesSource?: PowerBIPbitFilesSource; - /** - * PowerBI secrets. - */ - scope?: string[]; - /** - * Tenant ID for PowerBI. - * - * Azure Directory (tenant) ID for service principal authentication. - * - * Directory (tenant) ID from Azure Active Directory - */ - tenantId?: string; - /** - * Fetch the PowerBI metadata using admin APIs - */ - useAdminApis?: boolean; - /** - * Web Portal Virtual Directory Name. - */ - webPortalVirtualDirectory?: string; - /** - * Version of the Redash instance - */ - redashVersion?: string; - /** - * Choose between API or database connection fetch metadata from superset. - * - * Choose between Database connection or HDB User Store connection. - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Matillion Auth Configuration - */ - connection?: ConfigConnection; - /** - * Tableau API version. If not provided, the version will be used from the tableau server. - * - * Sigma API version. - * - * ThoughtSpot API version to use - * - * OpenMetadata server API version to use. - * - * Airbyte API version. - */ - apiVersion?: string; - /** - * Types of methods used to authenticate to the tableau instance - * - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - * - * Types of methods used to authenticate to the alation instance - * - * Authentication type to connect to Apache Ranger. - */ - authType?: AuthenticationTypeForTableau | NoConfigAuthenticationTypes; - /** - * Pagination limit used while querying the tableau metadata API for getting data sources - * - * Pagination limit used while querying the SAP ERP API for fetching the entities - * - * Pagination limit used for Alation APIs pagination - */ - paginationLimit?: number; - /** - * Proxy URL for the tableau server. If not provided, the hostPort will be used. This is - * used to generate the dashboard & Chart URL. - */ - proxyURL?: string; - /** - * Tableau Site Name. - * - * SharePoint site name - */ - siteName?: string; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; - /** - * Boolean marking if we need to verify the SSL certs for Grafana. Default to True. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Boolean marking if we need to verify the SSL certs for KafkaConnect REST API. True by - * default. - */ - verifySSL?: boolean | VerifySSL; - /** - * Access Token for Mode Dashboard - * - * Access token to connect to DOMO - */ - accessToken?: string; - /** - * Access Token Password for Mode Dashboard - */ - accessTokenPassword?: string; - /** - * Filter query parameter for some of the Mode API calls - */ - filterQueryParam?: string; - /** - * Mode Workspace Name - */ - workspaceName?: string; - connectionOptions?: { [key: string]: string }; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - * - * Secret token to connect to DOMO - */ - secretToken?: string; - /** - * AWS Account ID - */ - awsAccountId?: string; - awsConfig?: AWSCredentials; - /** - * The authentication method that the user uses to sign in. - */ - identityType?: IdentityType; - /** - * The Amazon QuickSight namespace that contains the dashboard IDs in this request ( To be - * provided when identityType is `ANONYMOUS` ) - */ - namespace?: string; - certificates?: QlikCertificatesBy; - /** - * Qlik Sense Base URL, used for genrating dashboard & chat url - */ - displayUrl?: string; - /** - * User Directory. - */ - userDirectory?: string; - /** - * User ID. - */ - userId?: string; - /** - * Validate Host Name - */ - validateHostName?: boolean; - /** - * The Project UUID for your Lightdash instance - */ - projectUUID?: string; - /** - * Use if your Lightdash instance is behind a proxy like (Cloud IAP) - */ - proxyAuthentication?: string; - /** - * The Space UUID for your Lightdash instance - */ - spaceUUID?: string; - /** - * Login Mode for Microstrategy's REST API connection. You can authenticate with one of the - * following authentication modes: `Standard (1)`, `Anonymous (8)`. Default will be - * `Standard (1)`. If you're using demo account for Microstrategy, it will be needed to - * authenticate through loginMode `8`. - */ - loginMode?: string; - /** - * MicroStrategy Project Name - * - * Project name to create the refreshToken. Can be anything - */ - projectName?: string; - /** - * Space types of Qlik Cloud to filter the dashboards ingested into the platform. - */ - spaceTypes?: SpaceType[]; - /** - * ThoughtSpot authentication configuration - */ - authentication?: Authenticationation; - /** - * Org ID for multi-tenant ThoughtSpot instances. This is applicable for ThoughtSpot Cloud - * only. - */ - orgId?: string; - /** - * Page size for pagination in API requests. Default is 100. - */ - pageSize?: number; - /** - * Whether to import Hex project categories as OpenMetadata tags - * - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Type of token to use for authentication - */ - tokenType?: TokenType; - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - * - * Azure Credentials - * - * GCP Credentials for Google Drive API - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Option to include policy tags as part of column description. - */ - includePolicyTags?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsQueryRunner?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in TimescaleDB. You can use databaseFilterPattern on top - * of this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - * - * Connection timeout in seconds. - */ - connectionTimeout?: number | number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - * - * Available sources to fetch metadata. - */ - configSource?: DeltaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - * - * Choose between Basic authentication (for self-hosted) or OAuth 2.0 client credentials - * (for Airbyte Cloud) - */ - auth?: Authentication | AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - * - * Custom OpenMetadata Classification name for TimescaleDB policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Consumer Key (Client ID) for OAuth 2.0 authentication. This is obtained from - * your Salesforce Connected App configuration. Required along with Consumer Secret for - * OAuth authentication. - */ - consumerKey?: string; - /** - * Salesforce Consumer Secret (Client Secret) for OAuth 2.0 authentication. This is obtained - * from your Salesforce Connected App configuration. Required along with Consumer Key for - * OAuth authentication. - */ - consumerSecret?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - * - * Snowplow BDP Organization ID - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token for username/password authentication. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - /** - * basic.auth.user.info schema registry config property, Client HTTP credentials in the form - * of username:password. - */ - basicAuthUserInfo?: string; - /** - * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - * - * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - */ - bootstrapServers?: string; - /** - * Confluent Kafka Consumer Config. From - * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - * - * Confluent Redpanda Consumer Config - */ - consumerConfig?: { [key: string]: any }; - /** - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - */ - consumerConfigSSL?: ConsumerConfigSSLClass; - /** - * sasl.mechanism Consumer Config property - */ - saslMechanism?: SaslMechanismType; - /** - * sasl.password consumer config property - */ - saslPassword?: string; - /** - * sasl.username consumer config property - */ - saslUsername?: string; - /** - * Confluent Kafka Schema Registry Config. From - * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient - * - * Confluent Redpanda Schema Registry Config. - */ - schemaRegistryConfig?: { [key: string]: any }; - /** - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ - schemaRegistrySSL?: ConsumerConfigSSLClass; - /** - * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get - * topic schema from registry. - */ - schemaRegistryTopicSuffixName?: string; - /** - * Confluent Kafka Schema Registry URL. - * - * Confluent Redpanda Schema Registry URL. - */ - schemaRegistryURL?: string; - /** - * security.protocol consumer config property - * - * Kafka security protocol config - */ - securityProtocol?: KafkaSecurityProtocol; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Enable encryption for the Amundsen Neo4j Connection. - */ - encrypted?: boolean; - /** - * Maximum connection lifetime for the Amundsen Neo4j Connection. - */ - maxConnectionLifeTime?: number; - /** - * Enable SSL validation for the Amundsen Neo4j Connection. - */ - validateSSL?: boolean; - /** - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - /** - * List of entities that you need to reindex - */ - entities?: string[]; - recreateIndex?: boolean; - runMode?: RunMode; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ConfigElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * service type of the data source. - */ - databaseServiceName?: string[]; - /** - * Name of the Entity Type available in Atlas. - */ - entity_type?: string; - /** - * service type of the messaging source - * - * Name of the Kafka Messaging Service associated with this Firehose Pipeline Service. e.g. - * local_kafka - * - * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. - * e.g. local_kafka - */ - messagingServiceName?: string[] | string; - /** - * Custom OpenMetadata Classification name for alation tags. - */ - alationTagClassificationName?: string; - /** - * Specifies if hidden datasources should be included while ingesting. - */ - includeHiddenDatasources?: boolean; - /** - * Specifies if undeployed datasources should be included while ingesting. - */ - includeUndeployedDatasources?: boolean; - /** - * Specifies if Dashboards are to be ingested while running the ingestion job. - */ - ingestDashboards?: boolean; - /** - * Specifies if Datasources are to be ingested while running the ingestion job. - */ - ingestDatasources?: boolean; - /** - * Specifies if Domains are to be ingested while running the ingestion job. - */ - ingestDomains?: boolean; - /** - * Specifies if Knowledge Articles are to be ingested while running the ingestion job. - */ - ingestKnowledgeArticles?: boolean; - /** - * Specifies if Users and Groups are to be ingested while running the ingestion job. - */ - ingestUsersAndGroups?: boolean; - datasourceLinks?: { [key: string]: string }; - /** - * Regex to only include/exclude domains that match the pattern. - */ - domainFilterPattern?: FilterPattern; - /** - * Enable enrichment of existing OpenMetadata assets with Collibra metadata (descriptions, - * tags, owners). When enabled, the connector will match Collibra assets to OpenMetadata - * entities and apply metadata without creating new assets. - */ - enableEnrichment?: boolean; - /** - * Regex to only include/exclude glossaries that match the pattern. - */ - glossaryFilterPattern?: FilterPattern; - /** - * Pipeline Service Number Of Status - */ - numberOfStatus?: number; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Underlying database connection - */ - databaseConnection?: DatabaseConnectionClass; - /** - * Underlying storage connection - */ - packageConnection?: S3Connection | string; - /** - * Fivetran API Secret. - */ - apiSecret?: string; - /** - * Fivetran API Limit For Pagination. - */ - limit?: number; - /** - * URL to the Dagster instance - * - * DBT cloud Access URL. - */ - host?: string; - /** - * Connection Time Limit Between OM and Dagster Graphql API in second - */ - timeout?: number; - /** - * We support username/password or client certificate authentication - */ - nifiConfig?: NifiCredentialsConfiguration; - /** - * Number of days to look back when fetching lineage data from Databricks system tables - * (system.access.table_lineage and system.access.column_lineage). Default is 90 days. - */ - lineageLookBackDays?: number; - /** - * Spline UI Host & Port. - */ - uiHostPort?: string; - /** - * service type of the messaging source - */ - brokersUrl?: string; - /** - * consumer group name - */ - consumerGroupName?: string; - /** - * initial Kafka consumer offset - */ - consumerOffsets?: InitialConsumerOffsets; - /** - * max allowed wait time - */ - poolTimeout?: number; - /** - * SASL Configuration details. - */ - saslConfig?: SASLClientConfig; - /** - * max allowed inactivity time - */ - sessionTimeout?: number; - /** - * topic from where Open lineage events will be pulled - */ - topicName?: string; - /** - * We support username/password or No Authentication - */ - KafkaConnectConfig?: UsernamePasswordAuthentication; - /** - * ID of your DBT cloud account - */ - accountId?: string; - /** - * DBT cloud Metadata API URL. - */ - discoveryAPI?: string; - /** - * List of IDs of your DBT cloud jobs seperated by comma `,` - */ - jobIds?: string[]; - /** - * Number of runs to fetch from DBT cloud - */ - numberOfRuns?: number; - /** - * List of IDs of your DBT cloud projects seperated by comma `,` - */ - projectIds?: string[]; - /** - * The name of your azure data factory. - */ - factory_name?: string; - /** - * The name of your resource group the data factory is associated with. - */ - resource_group_name?: string; - /** - * Number of days in the past to filter pipeline runs. - */ - run_filter_days?: number; - /** - * The azure subscription identifier. - */ - subscription_id?: string; - /** - * Cloud provider where Snowplow is deployed - */ - cloudProvider?: CloudProvider; - /** - * Path to pipeline configuration files for Community deployment - */ - configPath?: string; - /** - * Snowplow Console URL for BDP deployment - */ - consoleUrl?: string; - /** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - */ - deployment?: SnowplowDeployment; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Mlflow Model registry backend. E.g., - * mysql+pymysql://mlflow:password@localhost:3307/experiments - */ - registryUri?: string; - /** - * Mlflow Experiment tracking URI. E.g., http://localhost:5000 - */ - trackingUri?: string; - /** - * location/region of google cloud project - */ - location?: string; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - /** - * Console EndPoint URL for S3-compatible services - */ - consoleEndpointURL?: string; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Connection Timeout in Seconds - */ - connectionTimeoutSecs?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Email to impersonate using domain-wide delegation - */ - delegatedEmail?: string; - /** - * Regex to only include/exclude directories that matches the pattern. - */ - directoryFilterPattern?: FilterPattern; - /** - * Specific shared drive ID to connect to - * - * SharePoint drive ID. If not provided, default document library will be used - */ - driveId?: string; - /** - * Regex to only include/exclude files that matches the pattern. - */ - fileFilterPattern?: FilterPattern; - /** - * Extract metadata only for Google Sheets files - */ - includeGoogleSheets?: boolean; - /** - * Include shared/team drives in metadata extraction - */ - includeTeamDrives?: boolean; - /** - * Regex to only include/exclude spreadsheets that matches the pattern. - */ - spreadsheetFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude worksheets that matches the pattern. - */ - worksheetFilterPattern?: FilterPattern; - /** - * SharePoint site URL - */ - siteUrl?: string; - [property: string]: any; -} - -/** - * We support username/password or No Authentication - * - * username/password auth - */ -export interface UsernamePasswordAuthentication { - /** - * KafkaConnect password to authenticate to the API. - */ - password?: string; - /** - * KafkaConnect user to authenticate to the API. - */ - username?: string; -} - -/** - * Choose between Basic authentication (for self-hosted) or OAuth 2.0 client credentials - * (for Airbyte Cloud) - * - * Username and password authentication - * - * OAuth 2.0 client credentials authentication for Airbyte Cloud - */ -export interface Authentication { - /** - * Password to connect to Airbyte. - */ - password?: string; - /** - * Username to connect to Airbyte. - */ - username?: string; - /** - * Client ID for the application registered in Airbyte. - */ - clientId?: string; - /** - * Client Secret for the application registered in Airbyte. - */ - clientSecret?: string; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Types of methods used to authenticate to the tableau instance - * - * Basic Auth Credentials - * - * Access Token Auth Credentials - * - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - * - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * API Access Token Auth Credentials - * - * Basic Auth Configuration for ElasticSearch - * - * SSL Certificates By Path - * - * AWS credentials configs. - * - * Authentication type to connect to Apache Ranger. - * - * Configuration for connecting to Ranger Basic Auth. - */ -export interface AuthenticationTypeForTableau { - /** - * Password to access the service. - * - * Password to connect to source. - * - * Elastic Search Password for Login - * - * Ranger password to authenticate to the API. - */ - password?: string; - /** - * Username to access the service. - * - * Elastic Search Username for Login - * - * Ranger user to authenticate to the API. - */ - username?: string; - /** - * Personal Access Token Name. - */ - personalAccessTokenName?: string; - /** - * Personal Access Token Secret. - */ - personalAccessTokenSecret?: string; - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; - /** - * Access Token for the API - */ - accessToken?: string; - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -/** - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * Basic Auth Credentials - * - * API Access Token Auth Credentials - */ -export interface Authenticationation { - /** - * Password to access the service. - */ - password?: string; - /** - * Username to access the service. - */ - username?: string; - /** - * Access Token for the API - */ - accessToken?: string; -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: AuthenticationEnum; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum AuthenticationEnum { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: AWSCredentialsClass | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AWSCredentialsClass { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Qlik Authentication Certificate By Values - * - * Qlik Authentication Certificate File Path - */ -export interface QlikCertificatesBy { - sslConfig?: ConsumerConfigSSLClass; - /** - * Client Certificate - */ - clientCertificate?: string; - /** - * Client Key Certificate. - */ - clientKeyCertificate?: string; - /** - * Root Certificate. - */ - rootCertificate?: string; - [property: string]: any; -} - -/** - * Cloud provider where Snowplow is deployed - */ -export enum CloudProvider { - Aws = "AWS", - Azure = "Azure", - Gcp = "GCP", -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface DeltaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConfigSourceConnection; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: Credentials; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConfigSourceConnection { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Choose between API or database connection fetch metadata from superset. - * - * Superset API Connection Config - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - * - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Lineage Backend Connection Config - * - * SQLite Database Connection Config - * - * Matillion Auth Configuration - * - * Matillion ETL Auth Config. - */ -export interface ConfigConnection { - /** - * Password for Superset. - * - * Password to connect to Hana. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to the Matillion. - */ - password?: string; - /** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ - provider?: Provider; - /** - * SSL Configuration details. - */ - sslConfig?: ConnectionSSLConfig; - /** - * Username for Superset. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to Hana. This user should have privileges to read all the metadata. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to the Matillion. This user should have privileges to read all the - * metadata in Matillion. - */ - username?: string; - verifySSL?: VerifySSL; - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - * - * Host and port of the Hana service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Matillion Host - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: ConnectionScheme; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: ConnectionType; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - supportsViewLineageExtraction?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ -export enum Provider { - DB = "db", - LDAP = "ldap", -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum ConnectionScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", - SqlitePysqlite = "sqlite+pysqlite", -} - -/** - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Configuration for OpenMetadata Server - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ -export interface ConnectionSSLConfig { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum ConnectionType { - Backend = "Backend", - MatillionETL = "MatillionETL", - Mysql = "Mysql", - Postgres = "Postgres", - SQLite = "SQLite", -} - -/** - * initial Kafka consumer offset - */ -export enum InitialConsumerOffsets { - Earliest = "earliest", - Latest = "latest", -} - -/** - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Underlying database connection - * - * Mssql Database Connection Config - */ -export interface DatabaseConnectionClass { - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Host and port of the MSSQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to MSSQL. - */ - password?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: MssqlScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: MssqlType; - /** - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - */ - username?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum MssqlScheme { - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", -} - -/** - * Service Type - * - * Service type. - */ -export enum MssqlType { - Mssql = "Mssql", -} - -/** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - * - * Snowplow deployment type - */ -export enum SnowplowDeployment { - Bdp = "BDP", - Community = "Community", -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ConfigElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Do not set any credentials. Note that credentials are required to extract .lkml views and - * their lineage. - * - * Credentials for a GitHub repository - * - * Credentials for a BitBucket repository - * - * Credentials for a Gitlab repository - */ -export interface NoGitCredentialsClass { - /** - * GitHub instance URL. For GitHub.com, use https://github.com - * - * BitBucket instance URL. For BitBucket Cloud, use https://bitbucket.org - * - * Gitlab instance URL. For Gitlab.com, use https://gitlab.com - */ - gitHostURL?: string; - repositoryName?: string; - repositoryOwner?: string; - token?: string; - /** - * Credentials Type - */ - type?: NoGitCredentialsType; - /** - * Main production branch of the repository. E.g., `main` - */ - branch?: string; -} - -/** - * Credentials Type - * - * GitHub Credentials type - * - * BitBucket Credentials type - * - * Gitlab Credentials type - */ -export enum NoGitCredentialsType { - BitBucket = "BitBucket", - GitHub = "GitHub", - Gitlab = "Gitlab", -} - -/** - * The authentication method that the user uses to sign in. - */ -export enum IdentityType { - Anonymous = "ANONYMOUS", - Iam = "IAM", - Quicksight = "QUICKSIGHT", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: ConsumerConfigSSLClass; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * We support username/password or client certificate authentication - * - * Configuration for connecting to Nifi Basic Auth. - * - * Configuration for connecting to Nifi Client Certificate Auth. - */ -export interface NifiCredentialsConfiguration { - /** - * Nifi password to authenticate to the API. - */ - password?: string; - /** - * Nifi user to authenticate to the API. - */ - username?: string; - /** - * Boolean marking if we need to verify the SSL certs for Nifi. False by default. - */ - verifySSL?: boolean; - /** - * Path to the root CA certificate - */ - certificateAuthorityPath?: string; - /** - * Path to the client certificate - */ - clientCertificatePath?: string; - /** - * Path to the client key - */ - clientkeyPath?: string; -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * S3 Connection. - */ -export interface S3Connection { - awsConfig: AWSCredentials; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Console EndPoint URL for S3-compatible services - */ - consoleEndpointURL?: string; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: S3Type; -} - -/** - * Service Type - * - * S3 service type - */ -export enum S3Type { - S3 = "S3", -} - -/** - * Source to get the .pbit files to extract lineage information - * - * Local config source where no extra information needs to be sent. - * - * Azure storage config for pbit files - * - * GCS storage config for pbit files - * - * S3 storage config for pbit files - */ -export interface PowerBIPbitFilesSource { - /** - * Directory path for the pbit files - */ - path?: string; - /** - * pbit File Configuration type - */ - pbitFileConfigType?: PbitFileConfigType; - /** - * Path of the folder where the .pbit files will be unzipped and datamodel schema will be - * extracted - */ - pbitFilesExtractDir?: string; - prefixConfig?: BucketDetails; - securityConfig?: Credentials; -} - -/** - * pbit File Configuration type - */ -export enum PbitFileConfigType { - Azure = "azure", - Gcs = "gcs", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the .pbit files are stored - */ -export interface BucketDetails { - /** - * Name of the bucket where the .pbit files are stored - */ - bucketName?: string; - /** - * Path of the folder where the .pbit files are stored - */ - objectPrefix?: string; -} - -/** - * This schema publisher run modes. - */ -export enum RunMode { - Batch = "batch", - Stream = "stream", -} - -/** - * SASL Configuration details. - * - * SASL client configuration. - */ -export interface SASLClientConfig { - /** - * SASL security mechanism - */ - saslMechanism?: SaslMechanismType; - /** - * The SASL authentication password. - */ - saslPassword?: string; - /** - * The SASL authentication username. - */ - saslUsername?: string; -} - -/** - * sasl.mechanism Consumer Config property - * - * SASL Mechanism consumer config property - * - * SASL security mechanism - */ -export enum SaslMechanismType { - Gssapi = "GSSAPI", - Oauthbearer = "OAUTHBEARER", - Plain = "PLAIN", - ScramSHA256 = "SCRAM-SHA-256", - ScramSHA512 = "SCRAM-SHA-512", -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * security.protocol consumer config property - * - * Kafka security protocol config - */ -export enum KafkaSecurityProtocol { - Plaintext = "PLAINTEXT", - SSL = "SSL", - SaslPlaintext = "SASL_PLAINTEXT", - SaslSSL = "SASL_SSL", -} - -export enum SpaceType { - Data = "Data", - Managed = "Managed", - Personal = "Personal", - Shared = "Shared", -} - -/** - * SSL Configuration for OpenMetadata Server - * - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Config - */ -export interface SSLConfigObject { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; - /** - * SSL Certificates - */ - certificates?: SSLCertificates; - [property: string]: any; -} - -/** - * SSL Certificates - * - * SSL Configuration details. - * - * SSL Certificates By Path - * - * SSL Certificates By Values - */ -export interface SSLCertificates { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * CA Certificate Value - */ - caCertValue?: string; - /** - * Client Certificate Value - */ - clientCertValue?: string; - /** - * Private Key Value - */ - privateKeyValue?: string; - /** - * Staging Directory Path - */ - stagingDir?: string; -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * Type of token to use for authentication - */ -export enum TokenType { - Personal = "personal", - Workspace = "workspace", -} - -/** - * REST API Type - * - * REST API type - * - * Service Type - * - * Looker service type - * - * Metabase service type - * - * PowerBI service type - * - * PowerBIReportServer service type - * - * Redash service type - * - * Superset service type - * - * Tableau service type - * - * Mode service type - * - * Custom dashboard service type - * - * service type - * - * QuickSight service type - * - * Qlik sense service type - * - * Lightdash service type - * - * MicroStrategy service type - * - * Qlik Cloud service type - * - * Sigma service type - * - * ThoughtSpot service type - * - * Grafana service type - * - * Service type. - * - * Custom database service type - * - * Kafka service type - * - * Redpanda service type - * - * Custom messaging service type - * - * Amundsen service type - * - * Metadata to Elastic Search type - * - * OpenMetadata service type - * - * Collibra service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * S3 service type - * - * ADLS service type - * - * Gcs service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * ElasticSearch service type - * - * OpenSearch Type - * - * OpenSearch service type - * - * Custom search service type - * - * Apache Ranger service type - * - * Google Drive service type - * - * SharePoint service type - * - * Custom Drive service type - */ -export enum PurpleType { - Adls = "ADLS", - Airbyte = "Airbyte", - Airflow = "Airflow", - Alation = "Alation", - AlationSink = "AlationSink", - Amundsen = "Amundsen", - Athena = "Athena", - Atlas = "Atlas", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Collibra = "Collibra", - Couchbase = "Couchbase", - CustomDashboard = "CustomDashboard", - CustomDatabase = "CustomDatabase", - CustomDrive = "CustomDrive", - CustomMessaging = "CustomMessaging", - CustomMlModel = "CustomMlModel", - CustomPipeline = "CustomPipeline", - CustomSearch = "CustomSearch", - CustomStorage = "CustomStorage", - DBTCloud = "DBTCloud", - Dagster = "Dagster", - DataFactory = "DataFactory", - Databricks = "Databricks", - DatabricksPipeline = "DatabricksPipeline", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDashboard = "DomoDashboard", - DomoDatabase = "DomoDatabase", - DomoPipeline = "DomoPipeline", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - ElasticSearch = "ElasticSearch", - Epic = "Epic", - Exasol = "Exasol", - Fivetran = "Fivetran", - Flink = "Flink", - Gcs = "GCS", - Glue = "Glue", - GluePipeline = "GluePipeline", - GoogleDrive = "GoogleDrive", - Grafana = "Grafana", - Greenplum = "Greenplum", - Hex = "Hex", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - Kafka = "Kafka", - KafkaConnect = "KafkaConnect", - Kinesis = "Kinesis", - KinesisFirehose = "KinesisFirehose", - Lightdash = "Lightdash", - Looker = "Looker", - MariaDB = "MariaDB", - Matillion = "Matillion", - Metabase = "Metabase", - MetadataES = "MetadataES", - MicroStrategy = "MicroStrategy", - Mlflow = "Mlflow", - Mode = "Mode", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Nifi = "Nifi", - OpenLineage = "OpenLineage", - OpenMetadata = "OpenMetadata", - OpenSearch = "OpenSearch", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - PowerBI = "PowerBI", - PowerBIReportServer = "PowerBIReportServer", - Presto = "Presto", - QlikCloud = "QlikCloud", - QlikSense = "QlikSense", - QuickSight = "QuickSight", - REST = "Rest", - Ranger = "Ranger", - Redash = "Redash", - Redpanda = "Redpanda", - Redshift = "Redshift", - S3 = "S3", - SAS = "SAS", - SQLite = "SQLite", - SageMaker = "SageMaker", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SharePoint = "SharePoint", - Sigma = "Sigma", - SingleStore = "SingleStore", - Sklearn = "Sklearn", - Snowflake = "Snowflake", - Snowplow = "Snowplow", - Spark = "Spark", - Spline = "Spline", - Ssas = "SSAS", - Ssis = "SSIS", - Stitch = "Stitch", - Superset = "Superset", - Synapse = "Synapse", - Tableau = "Tableau", - Teradata = "Teradata", - ThoughtSpot = "ThoughtSpot", - Timescale = "Timescale", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - VertexAI = "VertexAI", - Vertica = "Vertica", - Wherescape = "Wherescape", -} - -/** - * No manifest file available. Ingestion would look for bucket-level metadata file instead - * - * Storage Metadata Manifest file path config. - * - * Storage Metadata Manifest file HTTP path config. - * - * Storage Metadata Manifest file S3 path config. - * - * Storage Metadata Manifest file ADLS path config. - * - * Storage Metadata Manifest file GCS path config. - */ -export interface StorageMetadataConfigurationSource { - /** - * Storage Metadata manifest file path to extract locations to ingest from. - */ - manifestFilePath?: string; - /** - * Storage Metadata manifest http file path to extract locations to ingest from. - */ - manifestHttpPath?: string; - prefixConfig?: StorageMetadataBucketDetails; - securityConfig?: Credentials; -} - -/** - * Details of the bucket where the storage metadata manifest file is stored - */ -export interface StorageMetadataBucketDetails { - /** - * Name of the top level container where the storage metadata file is stored - */ - containerName: string; - /** - * Path of the folder where the storage metadata file is stored. If the file is at the root, - * you can keep it empty. - */ - objectPrefix?: string; -} - -/** - * Pipeline type - * - * Database Source Config Metadata Pipeline type - * - * Database Source Config Usage Pipeline type - * - * Dashboard Source Config Metadata Pipeline type - * - * Messaging Source Config Metadata Pipeline type - * - * Profiler Source Config Pipeline type - * - * Pipeline Source Config Metadata Pipeline type - * - * MlModel Source Config Metadata Pipeline type - * - * Object Store Source Config Metadata Pipeline type - * - * Drive Source Config Metadata Pipeline type - * - * Search Source Config Metadata Pipeline type - * - * DBT Config Pipeline type - * - * Pipeline Source Config For Application Pipeline type. Nothing is required. - * - * Api Source Config Metadata Pipeline type - * - * Reverse Ingestion Config Pipeline type - */ -export enum FluffyType { - APIMetadata = "ApiMetadata", - Application = "Application", - AutoClassification = "AutoClassification", - DashboardMetadata = "DashboardMetadata", - DataInsight = "dataInsight", - DatabaseLineage = "DatabaseLineage", - DatabaseMetadata = "DatabaseMetadata", - DatabaseUsage = "DatabaseUsage", - Dbt = "DBT", - DriveMetadata = "DriveMetadata", - MessagingMetadata = "MessagingMetadata", - MetadataToElasticSearch = "MetadataToElasticSearch", - MlModelMetadata = "MlModelMetadata", - PipelineMetadata = "PipelineMetadata", - Profiler = "Profiler", - ReverseIngestion = "ReverseIngestion", - SearchMetadata = "SearchMetadata", - StorageMetadata = "StorageMetadata", - TestSuite = "TestSuite", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/application.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/application.ts deleted file mode 100644 index eaee951b3a9..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/application.ts +++ /dev/null @@ -1,1323 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * OpenMetadata Ingestion Framework definition for Applications, i.e., the YAML shape we - * require. - */ -export interface Application { - /** - * External Application configuration - */ - appConfig?: any[] | boolean | number | null | CollateAIAppConfig | string; - /** - * External Application Private configuration - */ - appPrivateConfig?: PrivateConfig; - /** - * Enable streaming logs to a remote log storage via the OpenMetadata Server - */ - enableStreamableLogs?: boolean; - /** - * Fully qualified name of ingestion pipeline, used to identify the current ingestion - * pipeline - */ - ingestionPipelineFQN?: string; - /** - * Unique identifier of pipeline run, used to identify the current pipeline run - */ - pipelineRunId?: string; - /** - * Source Python Class Name to run the application - */ - sourcePythonClass?: string; - /** - * General Workflow configuration, such as the OpenMetadata server connection and logging - * level - */ - workflowConfig: WorkflowConfig; -} - -/** - * Configuration for the CollateAI External Application. - * - * Configuration for the Automator External Application. - * - * This schema defines the Slack App Token Configuration - * - * Configuration for the Collate AI Quality Agent. - * - * No configuration needed to instantiate the Data Insights Pipeline. The logic is handled - * in the backend. - * - * Search Indexing App. - * - * Cache Warmup Application Configuration. - * - * Configuration for the AutoPilot Application. - */ -export interface CollateAIAppConfig { - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - filter?: string; - /** - * Patch the description if it is empty, instead of raising a suggestion - * - * Patch the tier if it is empty, instead of raising a suggestion - */ - patchIfEmpty?: boolean; - /** - * Application Type - */ - type?: Type; - /** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - */ - actions?: Action[]; - /** - * Entities selected to run the automation. - */ - resources?: Resource; - /** - * Bot Token - */ - botToken?: string; - /** - * User Token - */ - userToken?: string; - /** - * Whether the suggested tests should be active or not upon suggestion - * - * Whether the AutoPilot Workflow should be active or not. - */ - active?: boolean; - backfillConfiguration?: BackfillConfiguration; - /** - * Maximum number of events processed at a time (Default 100). - * - * Maximum number of events sent in a batch (Default 100). - * - * Number of entities to process in each batch. - */ - batchSize?: number; - moduleConfiguration?: ModuleConfiguration; - /** - * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property - * Type and are facing errors. Bear in mind that recreating the index will delete your - * DataAssets and a backfill will be needed. - */ - recreateDataAssetsIndex?: boolean; - sendToAdmins?: boolean; - sendToTeams?: boolean; - /** - * Enable automatic performance tuning based on cluster capabilities and database entity - * count - */ - autoTune?: boolean; - /** - * Number of threads to use for reindexing - * - * Number of parallel threads for processing entities and warming cache. - */ - consumerThreads?: number; - /** - * List of Entities to Reindex - * - * List of entity types to warm up in cache. Use 'all' to warm up all entity types. - */ - entities?: string[]; - /** - * Initial backoff time in milliseconds - */ - initialBackoff?: number; - /** - * Maximum backoff time in milliseconds - */ - maxBackoff?: number; - /** - * Maximum number of concurrent requests to the search index - */ - maxConcurrentRequests?: number; - /** - * Maximum number of retries for a failed request - */ - maxRetries?: number; - /** - * Maximum number of events sent in a batch (Default 100). - */ - payLoadSize?: number; - /** - * Number of threads to use for reindexing - */ - producerThreads?: number; - /** - * Queue Size to user internally for reindexing. - * - * Internal queue size for entity processing pipeline. - */ - queueSize?: number; - /** - * This schema publisher run modes. - */ - recreateIndex?: boolean; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Force cache warmup even if another instance is detected (use with caution). - */ - force?: boolean; - /** - * Enter the retention period for Activity Threads of type = 'Conversation' records in days - * (e.g., 30 for one month, 60 for two months). - */ - activityThreadsRetentionPeriod?: number; - /** - * Enter the retention period for change event records in days (e.g., 7 for one week, 30 for - * one month). - */ - changeEventRetentionPeriod?: number; - /** - * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two - * months). - */ - profileDataRetentionPeriod?: number; - /** - * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for - * two months). - */ - testCaseResultsRetentionPeriod?: number; - /** - * Service Entity Link for which to trigger the application. - */ - entityLink?: string; - [property: string]: any; -} - -/** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - * - * Apply Classification Tags to the selected assets. - * - * Remove Classification Tags Action Type - * - * Apply Glossary Terms to the selected assets. - * - * Remove Glossary Terms Action Type - * - * Add domains to the selected assets. - * - * Remove domains from the selected assets. - * - * Apply Tags to the selected assets. - * - * Add a Custom Property to the selected assets. - * - * Remove Owner Action Type - * - * Add an owner to the selected assets. - * - * Add Test Cases to the selected assets. - * - * Remove Test Cases Action Type - * - * Add owners to the selected assets. - * - * Remove Custom Properties Action Type - * - * Add a Data Product to the selected assets. - * - * Remove a Data Product to the selected assets. - * - * Propagate description, tags and glossary terms via lineage - * - * ML Tagging action configuration for external automator. - */ -export interface Action { - /** - * Apply tags to the children of the selected assets that match the criteria. E.g., columns, - * tasks, topic fields,... - * - * Remove tags from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply terms to the children of the selected assets that match the criteria. E.g., - * columns, tasks, topic fields,... - * - * Remove terms from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply the description to the children of the selected assets that match the criteria. - * E.g., columns, tasks, topic fields,... - * - * Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Add tests to the selected table columns - * - * Remove tests to the selected table columns - */ - applyToChildren?: string[]; - /** - * Update tags even if they are already defined in the asset. By default, incoming tags are - * merged with the existing ones. - * - * Update terms even if they are already defined in the asset. By default, incoming terms - * are merged with the existing ones. - * - * Update the domains even if they are defined in the asset. By default, we will only apply - * the domains to assets without domains. - * - * Update the description even if they are already defined in the asset. By default, we'll - * only add the descriptions to assets without the description set. - * - * Update the Custom Property even if it is defined in the asset. By default, we will only - * apply the owners to assets without the given Custom Property informed. - * - * Update the tier even if it is defined in the asset. By default, we will only apply the - * tier to assets without tier. - * - * Update the test even if it is defined in the asset. By default, we will only apply the - * test to assets without the existing test already existing. - * - * Update the owners even if it is defined in the asset. By default, we will only apply the - * owners to assets without owner. - * - * Update the Data Product even if the asset belongs to a different Domain. By default, we - * will only add the Data Product if the asset has no Domain, or it belongs to the same - * domain as the Data Product. - * - * Update descriptions, tags and Glossary Terms via lineage even if they are already defined - * in the asset. By default, descriptions are only updated if they are not already defined - * in the asset, and incoming tags are merged with the existing ones. - */ - overwriteMetadata?: boolean; - /** - * Classification Tags to apply (source must be 'Classification') - * - * Classification Tags to remove (source must be 'Classification') - */ - tags?: TierElement[]; - /** - * Application Type - */ - type: ActionType; - /** - * Remove tags from all the children and parent of the selected assets. - * - * Remove terms from all the children and parent of the selected assets. - * - * Remove descriptions from all the children and parent of the selected assets. - */ - applyToAll?: boolean; - /** - * Remove tags by its label type - * - * Remove terms by its label type - */ - labels?: LabelElement[]; - /** - * Glossary Terms to apply - * - * Glossary Terms to remove - */ - terms?: TierElement[]; - /** - * Domains to apply - */ - domains?: EntityReference[]; - /** - * Description to apply - */ - description?: string; - /** - * Owners to apply - * - * Custom Properties keys to remove - */ - customProperties?: any; - /** - * tier to apply - */ - tier?: TierElement; - /** - * Test Cases to apply - */ - testCases?: TestCaseDefinitions[]; - /** - * Remove all test cases - */ - removeAll?: boolean; - /** - * Test Cases to remove - */ - testCaseDefinitions?: string[]; - /** - * Owners to apply - */ - owners?: EntityReference[]; - /** - * Data Products to apply - * - * Data Products to remove - */ - dataProducts?: EntityReference[]; - /** - * Propagate the metadata to columns via column-level lineage. - */ - propagateColumnLevel?: boolean; - /** - * Propagate description through lineage - */ - propagateDescription?: boolean; - /** - * Propagate domains from the parent through lineage - */ - propagateDomains?: boolean; - /** - * Propagate glossary terms through lineage - */ - propagateGlossaryTerms?: boolean; - /** - * Propagate owner from the parent - */ - propagateOwner?: boolean; - /** - * Propagate the metadata to the parents (e.g., tables) via lineage. - */ - propagateParent?: boolean; - /** - * Propagate tags through lineage - */ - propagateTags?: boolean; - /** - * Propagate tier from the parent - */ - propagateTier?: boolean; - /** - * Number of levels to propagate lineage. If not set, it will propagate to all levels. - */ - propagationDepth?: number; - /** - * List of configurations to stop propagation based on conditions - */ - propagationStopConfigs?: PropagationStopConfig[]; -} - -/** - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Remove tags by its label type - * - * Remove terms by its label type - */ -export enum LabelElement { - Automated = "Automated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Configuration to stop lineage propagation based on conditions - */ -export interface PropagationStopConfig { - /** - * The metadata attribute to check for stopping propagation - */ - metadataAttribute: MetadataAttribute; - /** - * List of attribute values that will stop propagation when any of them is matched - */ - value: Array; -} - -/** - * The metadata attribute to check for stopping propagation - */ -export enum MetadataAttribute { - Description = "description", - Domains = "domains", - GlossaryTerms = "glossaryTerms", - Owner = "owner", - Tags = "tags", - Tier = "tier", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - * - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - */ -export interface TagLabel { - /** - * Description for the tag label. - * - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this tag. - * - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Link to the tag resource. - * - * Link to the entity resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType?: LabelTypeEnum; - /** - * Name of the tag or glossary term. - * - * Name of the entity instance. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source?: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state?: State; - style?: Style; - tagFQN?: string; - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id?: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type?: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelTypeEnum { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * Cover image configuration for the entity. - */ - coverImage?: CoverImage; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * Cover image configuration for the entity. - * - * Cover image configuration for an entity. This is used to display a banner or header image - * for entities like Domain, Glossary, Data Product, etc. - */ -export interface CoverImage { - /** - * Position of the cover image in CSS background-position format. Supports keywords (top, - * center, bottom) or pixel values (e.g., '20px 30px'). - */ - position?: string; - /** - * URL of the cover image. - */ - url?: string; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - */ -export interface TierElement { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelTypeEnum; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Minimum set of requirements to get a Test Case request ready - */ -export interface TestCaseDefinitions { - /** - * Compute the passed and failed row count for the test case. - */ - computePassedFailedRowCount?: boolean; - parameterValues?: TestCaseParameterValue[]; - /** - * Tags to apply - */ - tags?: TierElement[]; - /** - * Fully qualified name of the test definition. - */ - testDefinition?: string; - /** - * If the test definition supports it, use dynamic assertion to evaluate the test case. - */ - useDynamicAssertion?: boolean; - [property: string]: any; -} - -/** - * This schema defines the parameter values that can be passed for a Test Case. - */ -export interface TestCaseParameterValue { - /** - * name of the parameter. Must match the parameter names in testCaseParameterDefinition - */ - name?: string; - /** - * value to be passed for the Parameters. These are input from Users. We capture this in - * string and convert during the runtime. - */ - value?: string; - [property: string]: any; -} - -/** - * Application Type - * - * Add Tags action type. - * - * Remove Classification Tags Action Type. - * - * Add Terms action type. - * - * Remove Terms Action Type. - * - * Add Domain Action Type. - * - * Remove Domain Action Type - * - * Add Description Action Type. - * - * Add Custom Properties Action Type. - * - * Remove Description Action Type - * - * Add Tier Action Type. - * - * Remove Tier Action Type - * - * Add Test Case Action Type. - * - * Remove Test Case Action Type - * - * Add Owner Action Type. - * - * Remove Owner Action Type - * - * Remove Custom Properties Action Type. - * - * Add Data Products Action Type. - * - * Remove Data Products Action Type. - * - * Lineage propagation action type. - * - * ML PII Tagging action type. - */ -export enum ActionType { - AddCustomPropertiesAction = "AddCustomPropertiesAction", - AddDataProductAction = "AddDataProductAction", - AddDescriptionAction = "AddDescriptionAction", - AddDomainAction = "AddDomainAction", - AddOwnerAction = "AddOwnerAction", - AddTagsAction = "AddTagsAction", - AddTermsAction = "AddTermsAction", - AddTestCaseAction = "AddTestCaseAction", - AddTierAction = "AddTierAction", - LineagePropagationAction = "LineagePropagationAction", - MLTaggingAction = "MLTaggingAction", - RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", - RemoveDataProductAction = "RemoveDataProductAction", - RemoveDescriptionAction = "RemoveDescriptionAction", - RemoveDomainAction = "RemoveDomainAction", - RemoveOwnerAction = "RemoveOwnerAction", - RemoveTagsAction = "RemoveTagsAction", - RemoveTermsAction = "RemoveTermsAction", - RemoveTestCaseAction = "RemoveTestCaseAction", - RemoveTierAction = "RemoveTierAction", -} - -/** - * Backfill Configuration - */ -export interface BackfillConfiguration { - /** - * Enable Backfill for the configured dates - */ - enabled?: boolean; - /** - * Date for which the backfill will end - */ - endDate?: Date; - /** - * Date from which to start the backfill - */ - startDate?: Date; - [property: string]: any; -} - -/** - * Different Module Configurations - */ -export interface ModuleConfiguration { - /** - * App Analytics Module configuration - */ - appAnalytics: AppAnalyticsConfig; - /** - * Cost Analysis Insights Module configuration - */ - costAnalysis: CostAnalysisConfig; - /** - * Data Assets Insights Module configuration - */ - dataAssets: DataAssetsConfig; - /** - * Data Quality Insights Module configuration - */ - dataQuality: DataQualityConfig; -} - -/** - * App Analytics Module configuration - */ -export interface AppAnalyticsConfig { - /** - * If Enabled, App Analytics insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Cost Analysis Insights Module configuration - */ -export interface CostAnalysisConfig { - /** - * If Enabled, Cost Analysis insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Data Assets Insights Module configuration - */ -export interface DataAssetsConfig { - /** - * If Enabled, Data Asset insights will be populated when the App runs. - */ - enabled: boolean; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Defines the number of days the Data Assets Insights information will be kept. After it - * they will be deleted. - */ - retention?: number; - serviceFilter?: ServiceFilter; -} - -export interface ServiceFilter { - serviceName?: string; - serviceType?: string; -} - -/** - * Data Quality Insights Module configuration - */ -export interface DataQualityConfig { - /** - * If Enabled, Data Quality insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Entities selected to run the automation. - */ -export interface Resource { - /** - * Filter JSON tree to be used for rendering the filters in the UI. This comes from - * Immutable Tree type of react-awesome-query-builder. - */ - filterJsonTree?: string; - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - queryFilter?: string; - /** - * Type of the entity. E.g., 'table', 'chart',... - */ - type?: string[]; - [property: string]: any; -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Application Type - * - * Application type. - */ -export enum Type { - AutoPilotApplication = "AutoPilotApplication", - Automator = "Automator", - CacheWarmup = "CacheWarmup", - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - CollateAITierAgent = "CollateAITierAgent", - DataInsights = "DataInsights", - DataInsightsReport = "DataInsightsReport", - SearchIndexing = "SearchIndexing", -} - -/** - * External Application Private configuration - * - * Private Configuration for the CollateAI External Application. - */ -export interface PrivateConfig { - /** - * Collate Server public URL. WAII will use this information to interact with the server. - * E.g., https://sandbox.getcollate.io - */ - collateURL?: string; - /** - * Limits for the CollateAI Application. - */ - limits?: AppLimitsConfig; - /** - * WAII API Token - */ - token?: string; - /** - * WAII API host URL - */ - waiiInstance?: string; - [property: string]: any; -} - -/** - * Limits for the CollateAI Application. - * - * Private Configuration for the App Limits. - */ -export interface AppLimitsConfig { - /** - * The records of the limits. - */ - actions: { [key: string]: number }; - /** - * The start of this limit cycle. - */ - billingCycleStart: Date; -} - -/** - * General Workflow configuration, such as the OpenMetadata server connection and logging - * level - * - * Configuration for the entire Ingestion Workflow. - */ -export interface WorkflowConfig { - config?: { [key: string]: any }; - loggerLevel?: LogLevels; - openMetadataServerConfig: OpenMetadataConnection; - /** - * Control if we want to flag the workflow as failed if we encounter any processing errors. - */ - raiseOnError?: boolean; - /** - * The percentage of successfully processed records that must be achieved for the pipeline - * to be considered successful. Otherwise, the pipeline will be marked as failed. - */ - successThreshold?: number; -} - -/** - * Supported logging levels - */ -export enum LogLevels { - Debug = "DEBUG", - Error = "ERROR", - Info = "INFO", - Warn = "WARN", -} - -/** - * OpenMetadata Connection Config - */ -export interface OpenMetadataConnection { - /** - * OpenMetadata server API version to use. - */ - apiVersion?: string; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - */ - hostPort: string; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: Config; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: OpenmetadataType; - /** - * Flag to verify SSL Certificate for OpenMetadata Server. - */ - verifySSL?: VerifySSL; -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * SSL Configuration for OpenMetadata Server - * - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface Config { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Service Type - * - * OpenMetadata service type - */ -export enum OpenmetadataType { - OpenMetadata = "OpenMetadata", -} - -/** - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/applicationPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/applicationPipeline.ts deleted file mode 100644 index b3a923682a8..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/applicationPipeline.ts +++ /dev/null @@ -1,1010 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Application Pipeline Configuration. - */ -export interface ApplicationPipeline { - /** - * Application configuration - */ - appConfig?: any[] | boolean | number | null | CollateAIAppConfig | string; - /** - * Application private configuration - */ - appPrivateConfig?: PrivateConfig; - /** - * Source Python Class Name to run the application - */ - sourcePythonClass?: string; - /** - * Pipeline type - */ - type?: ApplicationConfigType; -} - -/** - * Configuration for the CollateAI External Application. - * - * Configuration for the Automator External Application. - * - * This schema defines the Slack App Token Configuration - * - * Configuration for the Collate AI Quality Agent. - * - * No configuration needed to instantiate the Data Insights Pipeline. The logic is handled - * in the backend. - * - * Search Indexing App. - * - * Cache Warmup Application Configuration. - * - * Configuration for the AutoPilot Application. - */ -export interface CollateAIAppConfig { - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - filter?: string; - /** - * Patch the description if it is empty, instead of raising a suggestion - * - * Patch the tier if it is empty, instead of raising a suggestion - */ - patchIfEmpty?: boolean; - /** - * Application Type - */ - type?: Type; - /** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - */ - actions?: Action[]; - /** - * Entities selected to run the automation. - */ - resources?: Resource; - /** - * Bot Token - */ - botToken?: string; - /** - * User Token - */ - userToken?: string; - /** - * Whether the suggested tests should be active or not upon suggestion - * - * Whether the AutoPilot Workflow should be active or not. - */ - active?: boolean; - backfillConfiguration?: BackfillConfiguration; - /** - * Maximum number of events processed at a time (Default 100). - * - * Maximum number of events sent in a batch (Default 100). - * - * Number of entities to process in each batch. - */ - batchSize?: number; - moduleConfiguration?: ModuleConfiguration; - /** - * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property - * Type and are facing errors. Bear in mind that recreating the index will delete your - * DataAssets and a backfill will be needed. - */ - recreateDataAssetsIndex?: boolean; - sendToAdmins?: boolean; - sendToTeams?: boolean; - /** - * Enable automatic performance tuning based on cluster capabilities and database entity - * count - */ - autoTune?: boolean; - /** - * Number of threads to use for reindexing - * - * Number of parallel threads for processing entities and warming cache. - */ - consumerThreads?: number; - /** - * List of Entities to Reindex - * - * List of entity types to warm up in cache. Use 'all' to warm up all entity types. - */ - entities?: string[]; - /** - * Initial backoff time in milliseconds - */ - initialBackoff?: number; - /** - * Maximum backoff time in milliseconds - */ - maxBackoff?: number; - /** - * Maximum number of concurrent requests to the search index - */ - maxConcurrentRequests?: number; - /** - * Maximum number of retries for a failed request - */ - maxRetries?: number; - /** - * Maximum number of events sent in a batch (Default 100). - */ - payLoadSize?: number; - /** - * Number of threads to use for reindexing - */ - producerThreads?: number; - /** - * Queue Size to user internally for reindexing. - * - * Internal queue size for entity processing pipeline. - */ - queueSize?: number; - /** - * This schema publisher run modes. - */ - recreateIndex?: boolean; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Force cache warmup even if another instance is detected (use with caution). - */ - force?: boolean; - /** - * Enter the retention period for Activity Threads of type = 'Conversation' records in days - * (e.g., 30 for one month, 60 for two months). - */ - activityThreadsRetentionPeriod?: number; - /** - * Enter the retention period for change event records in days (e.g., 7 for one week, 30 for - * one month). - */ - changeEventRetentionPeriod?: number; - /** - * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two - * months). - */ - profileDataRetentionPeriod?: number; - /** - * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for - * two months). - */ - testCaseResultsRetentionPeriod?: number; - /** - * Service Entity Link for which to trigger the application. - */ - entityLink?: string; - [property: string]: any; -} - -/** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - * - * Apply Classification Tags to the selected assets. - * - * Remove Classification Tags Action Type - * - * Apply Glossary Terms to the selected assets. - * - * Remove Glossary Terms Action Type - * - * Add domains to the selected assets. - * - * Remove domains from the selected assets. - * - * Apply Tags to the selected assets. - * - * Add a Custom Property to the selected assets. - * - * Remove Owner Action Type - * - * Add an owner to the selected assets. - * - * Add Test Cases to the selected assets. - * - * Remove Test Cases Action Type - * - * Add owners to the selected assets. - * - * Remove Custom Properties Action Type - * - * Add a Data Product to the selected assets. - * - * Remove a Data Product to the selected assets. - * - * Propagate description, tags and glossary terms via lineage - * - * ML Tagging action configuration for external automator. - */ -export interface Action { - /** - * Apply tags to the children of the selected assets that match the criteria. E.g., columns, - * tasks, topic fields,... - * - * Remove tags from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply terms to the children of the selected assets that match the criteria. E.g., - * columns, tasks, topic fields,... - * - * Remove terms from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply the description to the children of the selected assets that match the criteria. - * E.g., columns, tasks, topic fields,... - * - * Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Add tests to the selected table columns - * - * Remove tests to the selected table columns - */ - applyToChildren?: string[]; - /** - * Update tags even if they are already defined in the asset. By default, incoming tags are - * merged with the existing ones. - * - * Update terms even if they are already defined in the asset. By default, incoming terms - * are merged with the existing ones. - * - * Update the domains even if they are defined in the asset. By default, we will only apply - * the domains to assets without domains. - * - * Update the description even if they are already defined in the asset. By default, we'll - * only add the descriptions to assets without the description set. - * - * Update the Custom Property even if it is defined in the asset. By default, we will only - * apply the owners to assets without the given Custom Property informed. - * - * Update the tier even if it is defined in the asset. By default, we will only apply the - * tier to assets without tier. - * - * Update the test even if it is defined in the asset. By default, we will only apply the - * test to assets without the existing test already existing. - * - * Update the owners even if it is defined in the asset. By default, we will only apply the - * owners to assets without owner. - * - * Update the Data Product even if the asset belongs to a different Domain. By default, we - * will only add the Data Product if the asset has no Domain, or it belongs to the same - * domain as the Data Product. - * - * Update descriptions, tags and Glossary Terms via lineage even if they are already defined - * in the asset. By default, descriptions are only updated if they are not already defined - * in the asset, and incoming tags are merged with the existing ones. - */ - overwriteMetadata?: boolean; - /** - * Classification Tags to apply (source must be 'Classification') - * - * Classification Tags to remove (source must be 'Classification') - */ - tags?: TierElement[]; - /** - * Application Type - */ - type: ActionType; - /** - * Remove tags from all the children and parent of the selected assets. - * - * Remove terms from all the children and parent of the selected assets. - * - * Remove descriptions from all the children and parent of the selected assets. - */ - applyToAll?: boolean; - /** - * Remove tags by its label type - * - * Remove terms by its label type - */ - labels?: LabelElement[]; - /** - * Glossary Terms to apply - * - * Glossary Terms to remove - */ - terms?: TierElement[]; - /** - * Domains to apply - */ - domains?: EntityReference[]; - /** - * Description to apply - */ - description?: string; - /** - * Owners to apply - * - * Custom Properties keys to remove - */ - customProperties?: any; - /** - * tier to apply - */ - tier?: TierElement; - /** - * Test Cases to apply - */ - testCases?: TestCaseDefinitions[]; - /** - * Remove all test cases - */ - removeAll?: boolean; - /** - * Test Cases to remove - */ - testCaseDefinitions?: string[]; - /** - * Owners to apply - */ - owners?: EntityReference[]; - /** - * Data Products to apply - * - * Data Products to remove - */ - dataProducts?: EntityReference[]; - /** - * Propagate the metadata to columns via column-level lineage. - */ - propagateColumnLevel?: boolean; - /** - * Propagate description through lineage - */ - propagateDescription?: boolean; - /** - * Propagate domains from the parent through lineage - */ - propagateDomains?: boolean; - /** - * Propagate glossary terms through lineage - */ - propagateGlossaryTerms?: boolean; - /** - * Propagate owner from the parent - */ - propagateOwner?: boolean; - /** - * Propagate the metadata to the parents (e.g., tables) via lineage. - */ - propagateParent?: boolean; - /** - * Propagate tags through lineage - */ - propagateTags?: boolean; - /** - * Propagate tier from the parent - */ - propagateTier?: boolean; - /** - * Number of levels to propagate lineage. If not set, it will propagate to all levels. - */ - propagationDepth?: number; - /** - * List of configurations to stop propagation based on conditions - */ - propagationStopConfigs?: PropagationStopConfig[]; -} - -/** - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Remove tags by its label type - * - * Remove terms by its label type - */ -export enum LabelElement { - Automated = "Automated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Configuration to stop lineage propagation based on conditions - */ -export interface PropagationStopConfig { - /** - * The metadata attribute to check for stopping propagation - */ - metadataAttribute: MetadataAttribute; - /** - * List of attribute values that will stop propagation when any of them is matched - */ - value: Array; -} - -/** - * The metadata attribute to check for stopping propagation - */ -export enum MetadataAttribute { - Description = "description", - Domains = "domains", - GlossaryTerms = "glossaryTerms", - Owner = "owner", - Tags = "tags", - Tier = "tier", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - * - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - */ -export interface TagLabel { - /** - * Description for the tag label. - * - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this tag. - * - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Link to the tag resource. - * - * Link to the entity resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType?: LabelTypeEnum; - /** - * Name of the tag or glossary term. - * - * Name of the entity instance. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source?: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state?: State; - style?: Style; - tagFQN?: string; - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id?: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type?: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelTypeEnum { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * Cover image configuration for the entity. - */ - coverImage?: CoverImage; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * Cover image configuration for the entity. - * - * Cover image configuration for an entity. This is used to display a banner or header image - * for entities like Domain, Glossary, Data Product, etc. - */ -export interface CoverImage { - /** - * Position of the cover image in CSS background-position format. Supports keywords (top, - * center, bottom) or pixel values (e.g., '20px 30px'). - */ - position?: string; - /** - * URL of the cover image. - */ - url?: string; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - */ -export interface TierElement { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelTypeEnum; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Minimum set of requirements to get a Test Case request ready - */ -export interface TestCaseDefinitions { - /** - * Compute the passed and failed row count for the test case. - */ - computePassedFailedRowCount?: boolean; - parameterValues?: TestCaseParameterValue[]; - /** - * Tags to apply - */ - tags?: TierElement[]; - /** - * Fully qualified name of the test definition. - */ - testDefinition?: string; - /** - * If the test definition supports it, use dynamic assertion to evaluate the test case. - */ - useDynamicAssertion?: boolean; - [property: string]: any; -} - -/** - * This schema defines the parameter values that can be passed for a Test Case. - */ -export interface TestCaseParameterValue { - /** - * name of the parameter. Must match the parameter names in testCaseParameterDefinition - */ - name?: string; - /** - * value to be passed for the Parameters. These are input from Users. We capture this in - * string and convert during the runtime. - */ - value?: string; - [property: string]: any; -} - -/** - * Application Type - * - * Add Tags action type. - * - * Remove Classification Tags Action Type. - * - * Add Terms action type. - * - * Remove Terms Action Type. - * - * Add Domain Action Type. - * - * Remove Domain Action Type - * - * Add Description Action Type. - * - * Add Custom Properties Action Type. - * - * Remove Description Action Type - * - * Add Tier Action Type. - * - * Remove Tier Action Type - * - * Add Test Case Action Type. - * - * Remove Test Case Action Type - * - * Add Owner Action Type. - * - * Remove Owner Action Type - * - * Remove Custom Properties Action Type. - * - * Add Data Products Action Type. - * - * Remove Data Products Action Type. - * - * Lineage propagation action type. - * - * ML PII Tagging action type. - */ -export enum ActionType { - AddCustomPropertiesAction = "AddCustomPropertiesAction", - AddDataProductAction = "AddDataProductAction", - AddDescriptionAction = "AddDescriptionAction", - AddDomainAction = "AddDomainAction", - AddOwnerAction = "AddOwnerAction", - AddTagsAction = "AddTagsAction", - AddTermsAction = "AddTermsAction", - AddTestCaseAction = "AddTestCaseAction", - AddTierAction = "AddTierAction", - LineagePropagationAction = "LineagePropagationAction", - MLTaggingAction = "MLTaggingAction", - RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", - RemoveDataProductAction = "RemoveDataProductAction", - RemoveDescriptionAction = "RemoveDescriptionAction", - RemoveDomainAction = "RemoveDomainAction", - RemoveOwnerAction = "RemoveOwnerAction", - RemoveTagsAction = "RemoveTagsAction", - RemoveTermsAction = "RemoveTermsAction", - RemoveTestCaseAction = "RemoveTestCaseAction", - RemoveTierAction = "RemoveTierAction", -} - -/** - * Backfill Configuration - */ -export interface BackfillConfiguration { - /** - * Enable Backfill for the configured dates - */ - enabled?: boolean; - /** - * Date for which the backfill will end - */ - endDate?: Date; - /** - * Date from which to start the backfill - */ - startDate?: Date; - [property: string]: any; -} - -/** - * Different Module Configurations - */ -export interface ModuleConfiguration { - /** - * App Analytics Module configuration - */ - appAnalytics: AppAnalyticsConfig; - /** - * Cost Analysis Insights Module configuration - */ - costAnalysis: CostAnalysisConfig; - /** - * Data Assets Insights Module configuration - */ - dataAssets: DataAssetsConfig; - /** - * Data Quality Insights Module configuration - */ - dataQuality: DataQualityConfig; -} - -/** - * App Analytics Module configuration - */ -export interface AppAnalyticsConfig { - /** - * If Enabled, App Analytics insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Cost Analysis Insights Module configuration - */ -export interface CostAnalysisConfig { - /** - * If Enabled, Cost Analysis insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Data Assets Insights Module configuration - */ -export interface DataAssetsConfig { - /** - * If Enabled, Data Asset insights will be populated when the App runs. - */ - enabled: boolean; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Defines the number of days the Data Assets Insights information will be kept. After it - * they will be deleted. - */ - retention?: number; - serviceFilter?: ServiceFilter; -} - -export interface ServiceFilter { - serviceName?: string; - serviceType?: string; -} - -/** - * Data Quality Insights Module configuration - */ -export interface DataQualityConfig { - /** - * If Enabled, Data Quality insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Entities selected to run the automation. - */ -export interface Resource { - /** - * Filter JSON tree to be used for rendering the filters in the UI. This comes from - * Immutable Tree type of react-awesome-query-builder. - */ - filterJsonTree?: string; - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - queryFilter?: string; - /** - * Type of the entity. E.g., 'table', 'chart',... - */ - type?: string[]; - [property: string]: any; -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Application Type - * - * Application type. - */ -export enum Type { - AutoPilotApplication = "AutoPilotApplication", - Automator = "Automator", - CacheWarmup = "CacheWarmup", - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - CollateAITierAgent = "CollateAITierAgent", - DataInsights = "DataInsights", - DataInsightsReport = "DataInsightsReport", - SearchIndexing = "SearchIndexing", -} - -/** - * Application private configuration - * - * Private Configuration for the CollateAI External Application. - */ -export interface PrivateConfig { - /** - * Collate Server public URL. WAII will use this information to interact with the server. - * E.g., https://sandbox.getcollate.io - */ - collateURL?: string; - /** - * Limits for the CollateAI Application. - */ - limits?: AppLimitsConfig; - /** - * WAII API Token - */ - token?: string; - /** - * WAII API host URL - */ - waiiInstance?: string; - [property: string]: any; -} - -/** - * Limits for the CollateAI Application. - * - * Private Configuration for the App Limits. - */ -export interface AppLimitsConfig { - /** - * The records of the limits. - */ - actions: { [key: string]: number }; - /** - * The start of this limit cycle. - */ - billingCycleStart: Date; -} - -/** - * Pipeline type - * - * Pipeline Source Config For Application Pipeline type. Nothing is required. - */ -export enum ApplicationConfigType { - Application = "Application", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts deleted file mode 100644 index 9db9c3dcb04..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts +++ /dev/null @@ -1,6547 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * OpenMetadata Ingestion Framework definition. - */ -export interface Workflow { - /** - * Unique identifier that identifies this pipeline. - */ - id?: string; - /** - * Name that identifies this pipeline instance uniquely. - */ - name: string; - /** - * OpenMetadata Ingestion Workflow Config. - */ - openMetadataWorkflowConfig: OpenMetadataWorkflowConfig; -} - -/** - * OpenMetadata Ingestion Workflow Config. - */ -export interface OpenMetadataWorkflowConfig { - bulkSink?: BulkSink; - /** - * Enable streaming logs to a remote log storage via the OpenMetadata Server - */ - enableStreamableLogs?: boolean; - /** - * Fully qualified name of ingestion pipeline, used to identify the current ingestion - * pipeline - */ - ingestionPipelineFQN?: string; - /** - * Unique identifier of pipeline run, used to identify the current pipeline run - */ - pipelineRunId?: string; - processor?: Processor; - sink?: Sink; - source: Source; - stage?: Stage; - workflowConfig: WorkflowConfig; -} - -/** - * Configuration for BulkSink Component in the OpenMetadata Ingestion Framework. - */ -export interface BulkSink { - config?: { [key: string]: any }; - /** - * Type of BulkSink component ex: metadata-usage - */ - type: string; -} - -/** - * Configuration for Processor Component in the OpenMetadata Ingestion Framework. - */ -export interface Processor { - config?: { [key: string]: any }; - /** - * Type of processor component ex: pii-processor - */ - type: string; -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface Sink { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * Configuration for Source component in OpenMetadata Ingestion Framework. - */ -export interface Source { - /** - * Connection configuration for the source. ex: mysql , tableau connection. - */ - serviceConnection?: ServiceConnection; - /** - * Type of the source connector ex: mysql, snowflake, tableau etc.. - */ - serviceName?: string; - sourceConfig: SourceConfig; - /** - * Type of the source connector ex: mysql, snowflake, tableau etc.. - */ - type: string; -} - -/** - * Connection configuration for the source. ex: mysql , tableau connection. - * - * Supported services - * - * API Service Connection. - * - * Dashboard Connection. - * - * Database Connection. - * - * Metadata Service Connection. - * - * Pipeline Connection. - * - * MlModel Connection. - * - * Storage Connection. - * - * search Connection. - * - * Security Connection. - * - * Drive Connection. - */ -export interface ServiceConnection { - config?: ConfigObject; -} - -/** - * REST Connection Config - * - * Looker Connection Config - * - * Metabase Connection Config - * - * PowerBI Connection Config - * - * PowerBIReportServer Connection Config - * - * Redash Connection Config - * - * Superset Connection Config - * - * Tableau Connection Config - * - * Mode Connection Config - * - * Custom Dashboard Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Domo Dashboard Connection Config - * - * QuickSight Connection Config - * - * Qlik Sense Connection Config - * - * Lightdash Connection Config - * - * MicroStrategy Connection Config - * - * Qlik Cloud Connection Config - * - * Sigma Connection Config - * - * ThoughtSpot Connection Config - * - * Grafana Connection Config - * - * Hex Connection Config - * - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * TimescaleDB Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - * - * Kafka Connection Config - * - * Redpanda Connection Config - * - * Kinesis Connection Config - * - * Custom Messaging Service Connection to build a source that is not supported by - * OpenMetadata yet. - * - * Amundsen Connection Config - * - * Metadata to ElasticSearch Connection Config - * - * OpenMetadata Connection Config - * - * Atlas Connection Config - * - * Alation Connection Config - * - * Alation Sink Connection Config - * - * Collibra Connection Config - * - * Airflow Metadata Database Connection Config - * - * Wherescape Metadata Database Connection Config - * - * SSIS Metadata Database Connection Config - * - * Glue Pipeline Connection Config - * - * AWS Kinesis Firehose Pipeline Connection Config - * - * Airbyte Metadata Database Connection Config - * - * Fivetran Metadata Database Connection Config - * - * Flink Metadata Connection Config - * - * Dagster Metadata Database Connection Config - * - * Nifi Metadata Pipeline Connection Config - * - * Domo Pipeline Connection Config - * - * Custom Pipeline Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Spline Metadata Database Connection Config - * - * Spark Metadata Pipeline Connection Config - * - * OpenLineage Connection Config - * - * KafkaConnect Connection Config - * - * DBTCloud Connection Config - * - * Matillion Connection - * - * Azure Data Factory Connection Config - * - * Stitch Connection - * - * Snowplow Pipeline Connection Config - * - * MlFlow Connection Config - * - * Sklearn Connection Config - * - * Custom MlModel Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * SageMaker Connection Config - * - * Google VertexAI Connection Config - * - * S3 Connection. - * - * ADLS Connection. - * - * GCS Connection. - * - * Custom Storage Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * ElasticSearch Connection. - * - * OpenSearch Connection Config - * - * Custom Search Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * Apache Ranger Connection Config - * - * Google Drive Connection Config - * - * SharePoint Connection Config - * - * Custom Drive Connection to build a source that is not supported. - */ -export interface ConfigObject { - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Documentation URL for the schema. - */ - docURL?: string; - /** - * Open API Schema URL. - */ - openAPISchemaURL?: string; - /** - * Supports Metadata Extraction. - */ - supportsMetadataExtraction?: boolean; - /** - * Generated Token to connect to OpenAPI Schema. - * - * token to connect to Qlik Cloud. - * - * Hex API token for authentication. Can be personal or workspace token. - * - * To Connect to Dagster Cloud - * - * Generated Token to connect to Databricks. - * - * Generated Token to connect to DBTCloud. - * - * Token to connect to Stitch api doc - */ - token?: string; - /** - * REST API Type - * - * Service Type - * - * Custom dashboard service type - * - * Custom database service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * OpenSearch Type - * - * Custom search service type - */ - type?: PurpleType; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * User's Client ID. This user should have privileges to read all the metadata in Looker. - * - * client_id for PowerBI. - * - * Client ID for DOMO - * - * client_id for Sigma. - * - * Azure Application (client) ID for service principal authentication. - * - * Application (client) ID from Azure Active Directory - */ - clientId?: string; - /** - * User's Client Secret. - * - * clientSecret for PowerBI. - * - * clientSecret for Sigma. - * - * Azure Application client secret for service principal authentication. - * - * Application (client) secret from Azure Active Directory - */ - clientSecret?: string; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - */ - gitCredentials?: NoGitCredentialsClass | string; - /** - * URL to the Looker instance. - * - * Host and Port of the Metabase instance. - * - * Dashboard URL for PowerBI service. - * - * Dashboard URL for PowerBI Report Server. - * - * URL for the Redash instance - * - * URL for the superset instance. - * - * Tableau Server url. - * - * URL for the mode instance. - * - * URL for the Qlik instance. - * - * Address for your running Lightdash instance - * - * Host and Port of the MicroStrategy instance. - * - * Host and Port of the Qlik Cloud instance. - * - * Sigma API url. - * - * ThoughtSpot instance URL. Example: https://my-company.thoughtspot.cloud - * - * URL to the Grafana instance. - * - * Hex API URL. For Hex.tech cloud, use https://app.hex.tech - * - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the TimescaleDB service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - * - * Host and port of the Amundsen Neo4j Connection. This expect a URI format like: - * bolt://localhost:7687. - * - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - * - * Host and port of the Atlas service. - * - * Host and port of the Alation service. - * - * Host and port of the Collibra service. - * - * Pipeline Service Management/UI URI. - * - * Pipeline Service Management/UI URL. - * - * Spline REST Server Host & Port. - * - * KafkaConnect Service Management/UI URI. - * - * Host and port of the Stitch API host - * - * Host and port of the ElasticSearch service. - * - * Host and port of the OpenSearch service. - * - * Apache Ranger Admin URL. - */ - hostPort?: string; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * API token to connect to Metabase. Use this instead of username/password for token-based - * authentication. - * - * API key of the redash instance to access. - * - * The personal access token you can generate in the Lightdash app under the user settings - * - * Service Account Token to authenticate to the Grafana APIs. Use Service Account Tokens - * (format: glsa_xxxx) for authentication. Legacy API Keys are no longer supported by - * Grafana as of January 2025. Both self-hosted and Grafana Cloud are supported. Requires - * Admin role for full metadata extraction. - * - * API key to authenticate with the SAP ERP APIs. - * - * Fivetran API Key. - * - * API Key for Snowplow Console API - */ - apiKey?: string; - /** - * Password to connect to Metabase. Required for basic authentication. - * - * Password to connect to PowerBI report server. - * - * Password to connect to MicroStrategy. - * - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - * - * password to connect to the Amundsen Neo4j Connection. - * - * password to connect to the Atlas. - * - * Password to connect to the Collibra. - */ - password?: string; - /** - * Username to connect to Metabase. Required for basic authentication. - * - * Username to connect to PowerBI report server. - * - * Username for Redash - * - * Username to connect to MicroStrategy. This user should have privileges to read all the - * metadata in MicroStrategy. - * - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to TimescaleDB. This user should have privileges to read all the - * metadata in TimescaleDB. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to Salesforce. This user should have privileges to read all the - * metadata in Salesforce. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - * - * username to connect to the Amundsen Neo4j Connection. - * - * username to connect to the Atlas. This user should have privileges to read all the - * metadata in Atlas. - * - * Username to connect to the Collibra. This user should have privileges to read all the - * metadata in Collibra. - */ - username?: string; - /** - * API URL to call powerbi rest apis to extract metadata. Default to - * `https://api.powerbi.com`. You can provide youw own in case of different environment - */ - apiURL?: string; - /** - * Authority URI for the PowerBI service. - */ - authorityURI?: string; - /** - * Display Table Name from source instead of renamed table name for datamodel tables - */ - displayTableNameFromSource?: boolean; - /** - * Entity Limit set here will be used to paginate the PowerBi APIs - */ - pagination_entity_per_page?: number; - /** - * Source to get the .pbit files to extract lineage information - */ - pbitFilesSource?: PowerBIPbitFilesSource; - /** - * PowerBI secrets. - */ - scope?: string[]; - /** - * Tenant ID for PowerBI. - * - * Azure Directory (tenant) ID for service principal authentication. - * - * Directory (tenant) ID from Azure Active Directory - */ - tenantId?: string; - /** - * Fetch the PowerBI metadata using admin APIs - */ - useAdminApis?: boolean; - /** - * Web Portal Virtual Directory Name. - */ - webPortalVirtualDirectory?: string; - /** - * Version of the Redash instance - */ - redashVersion?: string; - /** - * Choose between API or database connection fetch metadata from superset. - * - * Choose between Database connection or HDB User Store connection. - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Matillion Auth Configuration - */ - connection?: ConfigConnection; - /** - * Tableau API version. If not provided, the version will be used from the tableau server. - * - * Sigma API version. - * - * ThoughtSpot API version to use - * - * OpenMetadata server API version to use. - * - * Airbyte API version. - */ - apiVersion?: string; - /** - * Types of methods used to authenticate to the tableau instance - * - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - * - * Types of methods used to authenticate to the alation instance - * - * Authentication type to connect to Apache Ranger. - */ - authType?: AuthenticationTypeForTableau | NoConfigAuthenticationTypes; - /** - * Pagination limit used while querying the tableau metadata API for getting data sources - * - * Pagination limit used while querying the SAP ERP API for fetching the entities - * - * Pagination limit used for Alation APIs pagination - */ - paginationLimit?: number; - /** - * Proxy URL for the tableau server. If not provided, the hostPort will be used. This is - * used to generate the dashboard & Chart URL. - */ - proxyURL?: string; - /** - * Tableau Site Name. - * - * SharePoint site name - */ - siteName?: string; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; - /** - * Boolean marking if we need to verify the SSL certs for Grafana. Default to True. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Boolean marking if we need to verify the SSL certs for KafkaConnect REST API. True by - * default. - */ - verifySSL?: boolean | VerifySSL; - /** - * Access Token for Mode Dashboard - * - * Access token to connect to DOMO - */ - accessToken?: string; - /** - * Access Token Password for Mode Dashboard - */ - accessTokenPassword?: string; - /** - * Filter query parameter for some of the Mode API calls - */ - filterQueryParam?: string; - /** - * Mode Workspace Name - */ - workspaceName?: string; - connectionOptions?: { [key: string]: string }; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - * - * Secret token to connect to DOMO - */ - secretToken?: string; - /** - * AWS Account ID - */ - awsAccountId?: string; - awsConfig?: AWSCredentials; - /** - * The authentication method that the user uses to sign in. - */ - identityType?: IdentityType; - /** - * The Amazon QuickSight namespace that contains the dashboard IDs in this request ( To be - * provided when identityType is `ANONYMOUS` ) - */ - namespace?: string; - certificates?: QlikCertificatesBy; - /** - * Qlik Sense Base URL, used for genrating dashboard & chat url - */ - displayUrl?: string; - /** - * User Directory. - */ - userDirectory?: string; - /** - * User ID. - */ - userId?: string; - /** - * Validate Host Name - */ - validateHostName?: boolean; - /** - * The Project UUID for your Lightdash instance - */ - projectUUID?: string; - /** - * Use if your Lightdash instance is behind a proxy like (Cloud IAP) - */ - proxyAuthentication?: string; - /** - * The Space UUID for your Lightdash instance - */ - spaceUUID?: string; - /** - * Login Mode for Microstrategy's REST API connection. You can authenticate with one of the - * following authentication modes: `Standard (1)`, `Anonymous (8)`. Default will be - * `Standard (1)`. If you're using demo account for Microstrategy, it will be needed to - * authenticate through loginMode `8`. - */ - loginMode?: string; - /** - * MicroStrategy Project Name - * - * Project name to create the refreshToken. Can be anything - */ - projectName?: string; - /** - * Space types of Qlik Cloud to filter the dashboards ingested into the platform. - */ - spaceTypes?: SpaceType[]; - /** - * ThoughtSpot authentication configuration - */ - authentication?: Authenticationation; - /** - * Org ID for multi-tenant ThoughtSpot instances. This is applicable for ThoughtSpot Cloud - * only. - */ - orgId?: string; - /** - * Page size for pagination in API requests. Default is 100. - */ - pageSize?: number; - /** - * Whether to import Hex project categories as OpenMetadata tags - * - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Type of token to use for authentication - */ - tokenType?: TokenType; - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - * - * Azure Credentials - * - * GCP Credentials for Google Drive API - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Option to include policy tags as part of column description. - */ - includePolicyTags?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsQueryRunner?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in TimescaleDB. You can use databaseFilterPattern on top - * of this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - * - * Connection timeout in seconds. - */ - connectionTimeout?: number | number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - * - * Available sources to fetch metadata. - */ - configSource?: DeltaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - * - * Choose between Basic authentication (for self-hosted) or OAuth 2.0 client credentials - * (for Airbyte Cloud) - */ - auth?: Authentication | AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - * - * Custom OpenMetadata Classification name for TimescaleDB policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Consumer Key (Client ID) for OAuth 2.0 authentication. This is obtained from - * your Salesforce Connected App configuration. Required along with Consumer Secret for - * OAuth authentication. - */ - consumerKey?: string; - /** - * Salesforce Consumer Secret (Client Secret) for OAuth 2.0 authentication. This is obtained - * from your Salesforce Connected App configuration. Required along with Consumer Key for - * OAuth authentication. - */ - consumerSecret?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - * - * Snowplow BDP Organization ID - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token for username/password authentication. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - /** - * basic.auth.user.info schema registry config property, Client HTTP credentials in the form - * of username:password. - */ - basicAuthUserInfo?: string; - /** - * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - * - * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - */ - bootstrapServers?: string; - /** - * Confluent Kafka Consumer Config. From - * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - * - * Confluent Redpanda Consumer Config - */ - consumerConfig?: { [key: string]: any }; - /** - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - */ - consumerConfigSSL?: ConsumerConfigSSLClass; - /** - * sasl.mechanism Consumer Config property - */ - saslMechanism?: SaslMechanismType; - /** - * sasl.password consumer config property - */ - saslPassword?: string; - /** - * sasl.username consumer config property - */ - saslUsername?: string; - /** - * Confluent Kafka Schema Registry Config. From - * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient - * - * Confluent Redpanda Schema Registry Config. - */ - schemaRegistryConfig?: { [key: string]: any }; - /** - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ - schemaRegistrySSL?: ConsumerConfigSSLClass; - /** - * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get - * topic schema from registry. - */ - schemaRegistryTopicSuffixName?: string; - /** - * Confluent Kafka Schema Registry URL. - * - * Confluent Redpanda Schema Registry URL. - */ - schemaRegistryURL?: string; - /** - * security.protocol consumer config property - * - * Kafka security protocol config - */ - securityProtocol?: KafkaSecurityProtocol; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Enable encryption for the Amundsen Neo4j Connection. - */ - encrypted?: boolean; - /** - * Maximum connection lifetime for the Amundsen Neo4j Connection. - */ - maxConnectionLifeTime?: number; - /** - * Enable SSL validation for the Amundsen Neo4j Connection. - */ - validateSSL?: boolean; - /** - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - /** - * List of entities that you need to reindex - */ - entities?: string[]; - recreateIndex?: boolean; - runMode?: RunMode; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ConfigElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * service type of the data source. - */ - databaseServiceName?: string[]; - /** - * Name of the Entity Type available in Atlas. - */ - entity_type?: string; - /** - * service type of the messaging source - * - * Name of the Kafka Messaging Service associated with this Firehose Pipeline Service. e.g. - * local_kafka - * - * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. - * e.g. local_kafka - */ - messagingServiceName?: string[] | string; - /** - * Custom OpenMetadata Classification name for alation tags. - */ - alationTagClassificationName?: string; - /** - * Specifies if hidden datasources should be included while ingesting. - */ - includeHiddenDatasources?: boolean; - /** - * Specifies if undeployed datasources should be included while ingesting. - */ - includeUndeployedDatasources?: boolean; - /** - * Specifies if Dashboards are to be ingested while running the ingestion job. - */ - ingestDashboards?: boolean; - /** - * Specifies if Datasources are to be ingested while running the ingestion job. - */ - ingestDatasources?: boolean; - /** - * Specifies if Domains are to be ingested while running the ingestion job. - */ - ingestDomains?: boolean; - /** - * Specifies if Knowledge Articles are to be ingested while running the ingestion job. - */ - ingestKnowledgeArticles?: boolean; - /** - * Specifies if Users and Groups are to be ingested while running the ingestion job. - */ - ingestUsersAndGroups?: boolean; - datasourceLinks?: { [key: string]: string }; - /** - * Regex to only include/exclude domains that match the pattern. - */ - domainFilterPattern?: FilterPattern; - /** - * Enable enrichment of existing OpenMetadata assets with Collibra metadata (descriptions, - * tags, owners). When enabled, the connector will match Collibra assets to OpenMetadata - * entities and apply metadata without creating new assets. - */ - enableEnrichment?: boolean; - /** - * Regex to only include/exclude glossaries that match the pattern. - */ - glossaryFilterPattern?: FilterPattern; - /** - * Pipeline Service Number Of Status - */ - numberOfStatus?: number; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Underlying database connection - */ - databaseConnection?: DatabaseConnectionClass; - /** - * Underlying storage connection - */ - packageConnection?: S3Connection | string; - /** - * Fivetran API Secret. - */ - apiSecret?: string; - /** - * Fivetran API Limit For Pagination. - */ - limit?: number; - /** - * URL to the Dagster instance - * - * DBT cloud Access URL. - */ - host?: string; - /** - * Connection Time Limit Between OM and Dagster Graphql API in second - */ - timeout?: number; - /** - * We support username/password or client certificate authentication - */ - nifiConfig?: NifiCredentialsConfiguration; - /** - * Number of days to look back when fetching lineage data from Databricks system tables - * (system.access.table_lineage and system.access.column_lineage). Default is 90 days. - */ - lineageLookBackDays?: number; - /** - * Spline UI Host & Port. - */ - uiHostPort?: string; - /** - * service type of the messaging source - */ - brokersUrl?: string; - /** - * consumer group name - */ - consumerGroupName?: string; - /** - * initial Kafka consumer offset - */ - consumerOffsets?: InitialConsumerOffsets; - /** - * max allowed wait time - */ - poolTimeout?: number; - /** - * SASL Configuration details. - */ - saslConfig?: SASLClientConfig; - /** - * max allowed inactivity time - */ - sessionTimeout?: number; - /** - * topic from where Open lineage events will be pulled - */ - topicName?: string; - /** - * We support username/password or No Authentication - */ - KafkaConnectConfig?: UsernamePasswordAuthentication; - /** - * ID of your DBT cloud account - */ - accountId?: string; - /** - * DBT cloud Metadata API URL. - */ - discoveryAPI?: string; - /** - * List of IDs of your DBT cloud jobs seperated by comma `,` - */ - jobIds?: string[]; - /** - * Number of runs to fetch from DBT cloud - */ - numberOfRuns?: number; - /** - * List of IDs of your DBT cloud projects seperated by comma `,` - */ - projectIds?: string[]; - /** - * The name of your azure data factory. - */ - factory_name?: string; - /** - * The name of your resource group the data factory is associated with. - */ - resource_group_name?: string; - /** - * Number of days in the past to filter pipeline runs. - */ - run_filter_days?: number; - /** - * The azure subscription identifier. - */ - subscription_id?: string; - /** - * Cloud provider where Snowplow is deployed - */ - cloudProvider?: CloudProvider; - /** - * Path to pipeline configuration files for Community deployment - */ - configPath?: string; - /** - * Snowplow Console URL for BDP deployment - */ - consoleUrl?: string; - /** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - */ - deployment?: SnowplowDeployment; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Mlflow Model registry backend. E.g., - * mysql+pymysql://mlflow:password@localhost:3307/experiments - */ - registryUri?: string; - /** - * Mlflow Experiment tracking URI. E.g., http://localhost:5000 - */ - trackingUri?: string; - /** - * location/region of google cloud project - */ - location?: string; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - /** - * Console EndPoint URL for S3-compatible services - */ - consoleEndpointURL?: string; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Connection Timeout in Seconds - */ - connectionTimeoutSecs?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Email to impersonate using domain-wide delegation - */ - delegatedEmail?: string; - /** - * Regex to only include/exclude directories that matches the pattern. - */ - directoryFilterPattern?: FilterPattern; - /** - * Specific shared drive ID to connect to - * - * SharePoint drive ID. If not provided, default document library will be used - */ - driveId?: string; - /** - * Regex to only include/exclude files that matches the pattern. - */ - fileFilterPattern?: FilterPattern; - /** - * Extract metadata only for Google Sheets files - */ - includeGoogleSheets?: boolean; - /** - * Include shared/team drives in metadata extraction - */ - includeTeamDrives?: boolean; - /** - * Regex to only include/exclude spreadsheets that matches the pattern. - */ - spreadsheetFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude worksheets that matches the pattern. - */ - worksheetFilterPattern?: FilterPattern; - /** - * SharePoint site URL - */ - siteUrl?: string; - [property: string]: any; -} - -/** - * We support username/password or No Authentication - * - * username/password auth - */ -export interface UsernamePasswordAuthentication { - /** - * KafkaConnect password to authenticate to the API. - */ - password?: string; - /** - * KafkaConnect user to authenticate to the API. - */ - username?: string; -} - -/** - * Regex to only fetch api collections with names matching the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex exclude or include charts that matches the pattern. - * - * Regex to exclude or include dashboards that matches the pattern. - * - * Regex exclude or include data models that matches the pattern. - * - * Regex to exclude or include projects that matches the pattern. - * - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - * - * Regex to only fetch topics that matches the pattern. - * - * Regex to only include/exclude domains that match the pattern. - * - * Regex to only include/exclude glossaries that match the pattern. - * - * Regex exclude pipelines. - * - * Regex to only fetch containers that matches the pattern. - * - * Regex to only fetch MlModels with names matching the pattern. - * - * Regex to only fetch search indexes that matches the pattern. - * - * Regex to only include/exclude directories that matches the pattern. - * - * Regex to only include/exclude files that matches the pattern. - * - * Regex to only include/exclude spreadsheets that matches the pattern. - * - * Regex to only include/exclude worksheets that matches the pattern. - * - * Regex to only fetch databases that matches the pattern. - * - * Regex to only fetch tables or databases that matches the pattern. - * - * Regex to only fetch stored procedures that matches the pattern. - * - * Regex exclude tables or databases that matches the pattern. - * - * Regex to only compute metrics for table that matches the given tag, tiers, gloassary - * pattern. - * - * Regex to only fetch tags that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Choose between Basic authentication (for self-hosted) or OAuth 2.0 client credentials - * (for Airbyte Cloud) - * - * Username and password authentication - * - * OAuth 2.0 client credentials authentication for Airbyte Cloud - */ -export interface Authentication { - /** - * Password to connect to Airbyte. - */ - password?: string; - /** - * Username to connect to Airbyte. - */ - username?: string; - /** - * Client ID for the application registered in Airbyte. - */ - clientId?: string; - /** - * Client Secret for the application registered in Airbyte. - */ - clientSecret?: string; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Types of methods used to authenticate to the tableau instance - * - * Basic Auth Credentials - * - * Access Token Auth Credentials - * - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - * - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * API Access Token Auth Credentials - * - * Basic Auth Configuration for ElasticSearch - * - * SSL Certificates By Path - * - * AWS credentials configs. - * - * Authentication type to connect to Apache Ranger. - * - * Configuration for connecting to Ranger Basic Auth. - */ -export interface AuthenticationTypeForTableau { - /** - * Password to access the service. - * - * Password to connect to source. - * - * Elastic Search Password for Login - * - * Ranger password to authenticate to the API. - */ - password?: string; - /** - * Username to access the service. - * - * Elastic Search Username for Login - * - * Ranger user to authenticate to the API. - */ - username?: string; - /** - * Personal Access Token Name. - */ - personalAccessTokenName?: string; - /** - * Personal Access Token Secret. - */ - personalAccessTokenSecret?: string; - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; - /** - * Access Token for the API - */ - accessToken?: string; - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -/** - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * Basic Auth Credentials - * - * API Access Token Auth Credentials - */ -export interface Authenticationation { - /** - * Password to access the service. - */ - password?: string; - /** - * Username to access the service. - */ - username?: string; - /** - * Access Token for the API - */ - accessToken?: string; -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: AuthenticationEnum; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum AuthenticationEnum { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: AWSCredentialsClass | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AWSCredentialsClass { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Qlik Authentication Certificate By Values - * - * Qlik Authentication Certificate File Path - */ -export interface QlikCertificatesBy { - sslConfig?: ConsumerConfigSSLClass; - /** - * Client Certificate - */ - clientCertificate?: string; - /** - * Client Key Certificate. - */ - clientKeyCertificate?: string; - /** - * Root Certificate. - */ - rootCertificate?: string; - [property: string]: any; -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface ConsumerConfigSSLClass { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Cloud provider where Snowplow is deployed - */ -export enum CloudProvider { - Aws = "AWS", - Azure = "Azure", - Gcp = "GCP", -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface DeltaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConfigSourceConnection; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: Credentials; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConfigSourceConnection { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - * - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * AWS credentials configs. - */ -export interface Credentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Choose between API or database connection fetch metadata from superset. - * - * Superset API Connection Config - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - * - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Lineage Backend Connection Config - * - * SQLite Database Connection Config - * - * Matillion Auth Configuration - * - * Matillion ETL Auth Config. - */ -export interface ConfigConnection { - /** - * Password for Superset. - * - * Password to connect to Hana. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to the Matillion. - */ - password?: string; - /** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ - provider?: Provider; - /** - * SSL Configuration details. - */ - sslConfig?: ConnectionSSLConfig; - /** - * Username for Superset. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to Hana. This user should have privileges to read all the metadata. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to the Matillion. This user should have privileges to read all the - * metadata in Matillion. - */ - username?: string; - verifySSL?: VerifySSL; - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - * - * Host and port of the Hana service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Matillion Host - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: ConnectionScheme; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: ConnectionType; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - supportsViewLineageExtraction?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ -export enum Provider { - DB = "db", - LDAP = "ldap", -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum ConnectionScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", - SqlitePysqlite = "sqlite+pysqlite", -} - -/** - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - */ -export interface ConnectionSSLConfig { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum ConnectionType { - Backend = "Backend", - MatillionETL = "MatillionETL", - Mysql = "Mysql", - Postgres = "Postgres", - SQLite = "SQLite", -} - -/** - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * initial Kafka consumer offset - */ -export enum InitialConsumerOffsets { - Earliest = "earliest", - Latest = "latest", -} - -/** - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Underlying database connection - * - * Mssql Database Connection Config - */ -export interface DatabaseConnectionClass { - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Host and port of the MSSQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to MSSQL. - */ - password?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: MssqlScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: MssqlType; - /** - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - */ - username?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum MssqlScheme { - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", -} - -/** - * Service Type - * - * Service type. - */ -export enum MssqlType { - Mssql = "Mssql", -} - -/** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - * - * Snowplow deployment type - */ -export enum SnowplowDeployment { - Bdp = "BDP", - Community = "Community", -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ConfigElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Do not set any credentials. Note that credentials are required to extract .lkml views and - * their lineage. - * - * Credentials for a GitHub repository - * - * Credentials for a BitBucket repository - * - * Credentials for a Gitlab repository - */ -export interface NoGitCredentialsClass { - /** - * GitHub instance URL. For GitHub.com, use https://github.com - * - * BitBucket instance URL. For BitBucket Cloud, use https://bitbucket.org - * - * Gitlab instance URL. For Gitlab.com, use https://gitlab.com - */ - gitHostURL?: string; - repositoryName?: string; - repositoryOwner?: string; - token?: string; - /** - * Credentials Type - */ - type?: NoGitCredentialsType; - /** - * Main production branch of the repository. E.g., `main` - */ - branch?: string; -} - -/** - * Credentials Type - * - * GitHub Credentials type - * - * BitBucket Credentials type - * - * Gitlab Credentials type - */ -export enum NoGitCredentialsType { - BitBucket = "BitBucket", - GitHub = "GitHub", - Gitlab = "Gitlab", -} - -/** - * The authentication method that the user uses to sign in. - */ -export enum IdentityType { - Anonymous = "ANONYMOUS", - Iam = "IAM", - Quicksight = "QUICKSIGHT", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: ConsumerConfigSSLClass; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * We support username/password or client certificate authentication - * - * Configuration for connecting to Nifi Basic Auth. - * - * Configuration for connecting to Nifi Client Certificate Auth. - */ -export interface NifiCredentialsConfiguration { - /** - * Nifi password to authenticate to the API. - */ - password?: string; - /** - * Nifi user to authenticate to the API. - */ - username?: string; - /** - * Boolean marking if we need to verify the SSL certs for Nifi. False by default. - */ - verifySSL?: boolean; - /** - * Path to the root CA certificate - */ - certificateAuthorityPath?: string; - /** - * Path to the client certificate - */ - clientCertificatePath?: string; - /** - * Path to the client key - */ - clientkeyPath?: string; -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * S3 Connection. - */ -export interface S3Connection { - awsConfig: AWSCredentials; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Console EndPoint URL for S3-compatible services - */ - consoleEndpointURL?: string; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: S3Type; -} - -/** - * Service Type - * - * S3 service type - */ -export enum S3Type { - S3 = "S3", -} - -/** - * Source to get the .pbit files to extract lineage information - * - * Local config source where no extra information needs to be sent. - * - * Azure storage config for pbit files - * - * GCS storage config for pbit files - * - * S3 storage config for pbit files - */ -export interface PowerBIPbitFilesSource { - /** - * Directory path for the pbit files - */ - path?: string; - /** - * pbit File Configuration type - */ - pbitFileConfigType?: PbitFileConfigType; - /** - * Path of the folder where the .pbit files will be unzipped and datamodel schema will be - * extracted - */ - pbitFilesExtractDir?: string; - prefixConfig?: BucketDetails; - securityConfig?: Credentials; -} - -/** - * pbit File Configuration type - */ -export enum PbitFileConfigType { - Azure = "azure", - Gcs = "gcs", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the .pbit files are stored - */ -export interface BucketDetails { - /** - * Name of the bucket where the .pbit files are stored - */ - bucketName?: string; - /** - * Path of the folder where the .pbit files are stored - */ - objectPrefix?: string; -} - -/** - * This schema publisher run modes. - */ -export enum RunMode { - Batch = "batch", - Stream = "stream", -} - -/** - * SASL Configuration details. - * - * SASL client configuration. - */ -export interface SASLClientConfig { - /** - * SASL security mechanism - */ - saslMechanism?: SaslMechanismType; - /** - * The SASL authentication password. - */ - saslPassword?: string; - /** - * The SASL authentication username. - */ - saslUsername?: string; -} - -/** - * sasl.mechanism Consumer Config property - * - * SASL Mechanism consumer config property - * - * SASL security mechanism - */ -export enum SaslMechanismType { - Gssapi = "GSSAPI", - Oauthbearer = "OAUTHBEARER", - Plain = "PLAIN", - ScramSHA256 = "SCRAM-SHA-256", - ScramSHA512 = "SCRAM-SHA-512", -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * security.protocol consumer config property - * - * Kafka security protocol config - */ -export enum KafkaSecurityProtocol { - Plaintext = "PLAINTEXT", - SSL = "SSL", - SaslPlaintext = "SASL_PLAINTEXT", - SaslSSL = "SASL_SSL", -} - -export enum SpaceType { - Data = "Data", - Managed = "Managed", - Personal = "Personal", - Shared = "Shared", -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Config - */ -export interface SSLConfigObject { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; - /** - * SSL Certificates - */ - certificates?: SSLCertificates; - [property: string]: any; -} - -/** - * SSL Certificates - * - * SSL Configuration details. - * - * SSL Certificates By Path - * - * SSL Certificates By Values - */ -export interface SSLCertificates { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * CA Certificate Value - */ - caCertValue?: string; - /** - * Client Certificate Value - */ - clientCertValue?: string; - /** - * Private Key Value - */ - privateKeyValue?: string; - /** - * Staging Directory Path - */ - stagingDir?: string; -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * Type of token to use for authentication - */ -export enum TokenType { - Personal = "personal", - Workspace = "workspace", -} - -/** - * REST API Type - * - * REST API type - * - * Service Type - * - * Looker service type - * - * Metabase service type - * - * PowerBI service type - * - * PowerBIReportServer service type - * - * Redash service type - * - * Superset service type - * - * Tableau service type - * - * Mode service type - * - * Custom dashboard service type - * - * service type - * - * QuickSight service type - * - * Qlik sense service type - * - * Lightdash service type - * - * MicroStrategy service type - * - * Qlik Cloud service type - * - * Sigma service type - * - * ThoughtSpot service type - * - * Grafana service type - * - * Service type. - * - * Custom database service type - * - * Kafka service type - * - * Redpanda service type - * - * Custom messaging service type - * - * Amundsen service type - * - * Metadata to Elastic Search type - * - * OpenMetadata service type - * - * Collibra service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * S3 service type - * - * ADLS service type - * - * Gcs service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * ElasticSearch service type - * - * OpenSearch Type - * - * OpenSearch service type - * - * Custom search service type - * - * Apache Ranger service type - * - * Google Drive service type - * - * SharePoint service type - * - * Custom Drive service type - */ -export enum PurpleType { - Adls = "ADLS", - Airbyte = "Airbyte", - Airflow = "Airflow", - Alation = "Alation", - AlationSink = "AlationSink", - Amundsen = "Amundsen", - Athena = "Athena", - Atlas = "Atlas", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Collibra = "Collibra", - Couchbase = "Couchbase", - CustomDashboard = "CustomDashboard", - CustomDatabase = "CustomDatabase", - CustomDrive = "CustomDrive", - CustomMessaging = "CustomMessaging", - CustomMlModel = "CustomMlModel", - CustomPipeline = "CustomPipeline", - CustomSearch = "CustomSearch", - CustomStorage = "CustomStorage", - DBTCloud = "DBTCloud", - Dagster = "Dagster", - DataFactory = "DataFactory", - Databricks = "Databricks", - DatabricksPipeline = "DatabricksPipeline", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDashboard = "DomoDashboard", - DomoDatabase = "DomoDatabase", - DomoPipeline = "DomoPipeline", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - ElasticSearch = "ElasticSearch", - Epic = "Epic", - Exasol = "Exasol", - Fivetran = "Fivetran", - Flink = "Flink", - Gcs = "GCS", - Glue = "Glue", - GluePipeline = "GluePipeline", - GoogleDrive = "GoogleDrive", - Grafana = "Grafana", - Greenplum = "Greenplum", - Hex = "Hex", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - Kafka = "Kafka", - KafkaConnect = "KafkaConnect", - Kinesis = "Kinesis", - KinesisFirehose = "KinesisFirehose", - Lightdash = "Lightdash", - Looker = "Looker", - MariaDB = "MariaDB", - Matillion = "Matillion", - Metabase = "Metabase", - MetadataES = "MetadataES", - MicroStrategy = "MicroStrategy", - Mlflow = "Mlflow", - Mode = "Mode", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Nifi = "Nifi", - OpenLineage = "OpenLineage", - OpenMetadata = "OpenMetadata", - OpenSearch = "OpenSearch", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - PowerBI = "PowerBI", - PowerBIReportServer = "PowerBIReportServer", - Presto = "Presto", - QlikCloud = "QlikCloud", - QlikSense = "QlikSense", - QuickSight = "QuickSight", - REST = "Rest", - Ranger = "Ranger", - Redash = "Redash", - Redpanda = "Redpanda", - Redshift = "Redshift", - S3 = "S3", - SAS = "SAS", - SQLite = "SQLite", - SageMaker = "SageMaker", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SharePoint = "SharePoint", - Sigma = "Sigma", - SingleStore = "SingleStore", - Sklearn = "Sklearn", - Snowflake = "Snowflake", - Snowplow = "Snowplow", - Spark = "Spark", - Spline = "Spline", - Ssas = "SSAS", - Ssis = "SSIS", - Stitch = "Stitch", - Superset = "Superset", - Synapse = "Synapse", - Tableau = "Tableau", - Teradata = "Teradata", - ThoughtSpot = "ThoughtSpot", - Timescale = "Timescale", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - VertexAI = "VertexAI", - Vertica = "Vertica", - Wherescape = "Wherescape", -} - -/** - * Additional connection configuration. - */ -export interface SourceConfig { - config?: Pipeline; -} - -/** - * DatabaseService Metadata Pipeline Configuration. - * - * DatabaseService Query Usage Pipeline Configuration. - * - * DatabaseService Query Lineage Pipeline Configuration. - * - * DashboardService Metadata Pipeline Configuration. - * - * MessagingService Metadata Pipeline Configuration. - * - * DatabaseService Profiler Pipeline Configuration. - * - * DatabaseService AutoClassification & Auto Classification Pipeline Configuration. - * - * PipelineService Metadata Pipeline Configuration. - * - * MlModelService Metadata Pipeline Configuration. - * - * StorageService Metadata Pipeline Configuration. - * - * DriveService Metadata Pipeline Configuration. - * - * SearchService Metadata Pipeline Configuration. - * - * TestSuite Pipeline Configuration. - * - * Data Insight Pipeline Configuration. - * - * DBT Pipeline Configuration. - * - * Application Pipeline Configuration. - * - * ApiService Metadata Pipeline Configuration. - * - * Apply a set of operations on a service - */ -export interface Pipeline { - /** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Optional configuration to toggle the DDL Statements ingestion. - */ - includeDDL?: boolean; - /** - * Set the 'Include Owners' toggle to control whether to include owners to the ingested - * entity if the owner email matches with a user stored in the OM server as part of metadata - * ingestion. If the ingested entity already exists and has an owner, the owner will not be - * overwritten. - * - * Enabling a flag will replace the current owner with a new owner from the source during - * metadata ingestion, if the current owner is null. It is recommended to keep the flag - * enabled to obtain the owner information during the first metadata ingestion. - */ - includeOwners?: boolean; - /** - * Optional configuration to toggle the Stored Procedures ingestion. - */ - includeStoredProcedures?: boolean; - /** - * Optional configuration to turn off fetching metadata for tables. - */ - includeTables?: boolean; - /** - * Optional configuration to toggle the tags ingestion. - */ - includeTags?: boolean; - /** - * Optional configuration to turn off fetching metadata for views. - */ - includeViews?: boolean; - /** - * Use incremental Metadata extraction after the first execution. This is commonly done by - * getting the changes from Audit tables on the supporting databases. - */ - incremental?: IncrementalMetadataExtractionConfiguration; - /** - * Optional configuration to soft delete databases in OpenMetadata if the source databases - * are deleted. Also, if the database is deleted, all the associated entities like schemas, - * tables, views, stored procedures, lineage, etc., with that database will be deleted - */ - markDeletedDatabases?: boolean; - /** - * Optional configuration to soft delete schemas in OpenMetadata if the source schemas are - * deleted. Also, if the schema is deleted, all the associated entities like tables, views, - * stored procedures, lineage, etc., with that schema will be deleted - */ - markDeletedSchemas?: boolean; - /** - * Optional configuration to soft delete stored procedures in OpenMetadata if the source - * stored procedures are deleted. Also, if the stored procedures is deleted, all the - * associated entities like lineage, etc., with that stored procedures will be deleted - */ - markDeletedStoredProcedures?: boolean; - /** - * This is an optional configuration for enabling soft deletion of tables. When this option - * is enabled, only tables that have been deleted from the source will be soft deleted, and - * this will apply solely to the schema that is currently being ingested via the pipeline. - * Any related entities such as test suites or lineage information that were associated with - * those tables will also be deleted. - */ - markDeletedTables?: boolean; - /** - * Set the 'Override Metadata' toggle to control whether to override the existing metadata - * in the OpenMetadata server with the metadata fetched from the source. If the toggle is - * set to true, the metadata fetched from the source will override the existing metadata in - * the OpenMetadata server. If the toggle is set to false, the metadata fetched from the - * source will not override the existing metadata in the OpenMetadata server. This is - * applicable for fields like description, tags, owner and displayName - */ - overrideMetadata?: boolean; - /** - * Advanced configuration for managing owners at multiple hierarchy levels (Service, - * Database, Schema, Table) with custom mappings and inheritance rules. - */ - ownerConfig?: OwnerConfiguration; - /** - * Configuration to tune how far we want to look back in query logs to process Stored - * Procedures results. - * - * Configuration to tune how far we want to look back in query logs to process usage data. - * - * Configuration to tune how far we want to look back in query logs to process lineage data. - */ - queryLogDuration?: number; - /** - * Configuration to set the timeout for parsing the query in seconds. - */ - queryParsingTimeoutLimit?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only fetch tables or databases that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex exclude tables or databases that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Number of Threads to use in order to parallelize Table ingestion. - * - * Number of Threads to use in order to parallelize lineage ingestion. - * - * Number of Threads to use in order to parallelize Drive ingestion. - */ - threads?: number; - /** - * Pipeline type - */ - type?: FluffyType; - /** - * Regex will be applied on fully qualified name (e.g - * service_name.db_name.schema_name.table_name) instead of raw name (e.g. table_name) - * - * Regex will be applied on fully qualified name (e.g service_name.directory_name.file_name) - * instead of raw name (e.g. file_name) - */ - useFqnForFiltering?: boolean; - /** - * Configuration the condition to filter the query history. - */ - filterCondition?: string; - /** - * Configuration to process query cost - */ - processQueryCostAnalysis?: boolean; - /** - * Configuration to set the file path for query logs - */ - queryLogFilePath?: string; - /** - * Configuration to set the limit for query logs - */ - resultLimit?: number; - /** - * Temporary file name to store the query logs before processing. Absolute file path - * required. - */ - stageFileLocation?: string; - /** - * Set 'Cross Database Service Names' to process lineage with the database. - */ - crossDatabaseServiceNames?: string[]; - /** - * Handle Lineage for Snowflake Temporary and Transient Tables. - */ - enableTempTableLineage?: boolean; - /** - * Set the 'Incremental Lineage Processing' toggle to control whether to process lineage - * incrementally. - */ - incrementalLineageProcessing?: boolean; - /** - * Set the 'Override View Lineage' toggle to control whether to override the existing view - * lineage. - */ - overrideViewLineage?: boolean; - /** - * Configuration to set the timeout for parsing the query in seconds. - */ - parsingTimeoutLimit?: number; - /** - * Set the 'Process Cross Database Lineage' toggle to control whether to process table - * lineage across different databases. - */ - processCrossDatabaseLineage?: boolean; - /** - * Set the 'Process Query Lineage' toggle to control whether to process query lineage. - */ - processQueryLineage?: boolean; - /** - * Set the 'Process Stored ProcedureLog Lineage' toggle to control whether to process stored - * procedure lineage. - */ - processStoredProcedureLineage?: boolean; - /** - * Set the 'Process View Lineage' toggle to control whether to process view lineage. - */ - processViewLineage?: boolean; - /** - * Regex to only fetch stored procedures that matches the pattern. - */ - storedProcedureFilterPattern?: FilterPattern; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Optional configuration to toggle the ingestion of data models. - */ - includeDataModels?: boolean; - /** - * Optional Configuration to include/exclude draft dashboards. By default it will include - * draft dashboards - */ - includeDraftDashboard?: boolean; - /** - * Details required to generate Lineage - */ - lineageInformation?: LineageInformation; - /** - * Optional configuration to soft delete dashboards in OpenMetadata if the source dashboards - * are deleted. Also, if the dashboard is deleted, all the associated entities like lineage, - * etc., with that dashboard will be deleted - */ - markDeletedDashboards?: boolean; - /** - * Optional configuration to soft delete data models in OpenMetadata if the source data - * models are deleted. Also, if the data models is deleted, all the associated entities like - * lineage, etc., with that data models will be deleted - */ - markDeletedDataModels?: boolean; - /** - * Set the 'Override Lineage' toggle to control whether to override the existing lineage. - */ - overrideLineage?: boolean; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * Option to turn on/off generating sample data during metadata extraction. - */ - generateSampleData?: boolean; - /** - * Optional configuration to soft delete topics in OpenMetadata if the source topics are - * deleted. Also, if the topic is deleted, all the associated entities like sample data, - * lineage, etc., with that topic will be deleted - */ - markDeletedTopics?: boolean; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Regex to only compute metrics for table that matches the given tag, tiers, gloassary - * pattern. - */ - classificationFilterPattern?: FilterPattern; - /** - * Option to turn on/off column metric computation. If enabled, profiler will compute column - * level metrics. - */ - computeColumnMetrics?: boolean; - /** - * Option to turn on/off computing profiler metrics. - */ - computeMetrics?: boolean; - /** - * Option to turn on/off table metric computation. If enabled, profiler will compute table - * level metrics. - */ - computeTableMetrics?: boolean; - processingEngine?: ProcessingEngine; - /** - * Percentage of data or no. of rows used to compute the profiler metrics and run data - * quality tests - * - * Percentage of data or no. of rows we want to execute the profiler and tests on - */ - profileSample?: number; - profileSampleType?: ProfileSampleType; - /** - * Whether to randomize the sample data or not. - */ - randomizedSample?: boolean; - samplingMethodType?: SamplingMethodType; - /** - * Number of threads to use during metric computations - */ - threadCount?: number; - /** - * Profiler Timeout in Seconds - */ - timeoutSeconds?: number; - /** - * Use system tables to extract metrics. Metrics that cannot be gathered from system tables - * will use the default methods. Using system tables can be faster but requires gathering - * statistics before running (for example using the ANALYZE procedure). More information can - * be found in the documentation: https://docs.openmetadata.org/latest/profler - */ - useStatistics?: boolean; - /** - * Set the Confidence value for which you want the column to be tagged as PII. Confidence - * value ranges from 0 to 100. A higher number will yield less false positives but more - * false negatives. A lower number will yield more false positives but less false negatives. - */ - confidence?: number; - /** - * Optional configuration to automatically tag columns that might contain sensitive - * information - */ - enableAutoClassification?: boolean; - /** - * Number of sample rows to ingest when 'Generate Sample Data' is enabled - */ - sampleDataCount?: number; - /** - * Option to turn on/off storing sample data. If enabled, we will ingest sample data for - * each table. - */ - storeSampleData?: boolean; - /** - * Optional configuration to turn off fetching lineage from pipelines. - */ - includeLineage?: boolean; - /** - * Optional configuration to toggle whether the un-deployed pipelines should be ingested or - * not. If set to false, only deployed pipelines will be ingested. - */ - includeUnDeployedPipelines?: boolean; - /** - * Optional configuration to soft delete Pipelines in OpenMetadata if the source Pipelines - * are deleted. Also, if the Pipeline is deleted, all the associated entities like lineage, - * etc., with that Pipeline will be deleted - */ - markDeletedPipelines?: boolean; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete MlModels in OpenMetadata if the source MlModels are - * deleted. Also, if the MlModel is deleted, all the associated entities like lineage, etc., - * with that MlModels will be deleted - */ - markDeletedMlModels?: boolean; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete containers in OpenMetadata if the source containers - * are deleted. Also, if the topic is deleted, all the associated entities with that - * containers will be deleted - */ - markDeletedContainers?: boolean; - storageMetadataConfigSource?: StorageMetadataConfigurationSource; - /** - * Regex to only include/exclude directories that matches the pattern. - */ - directoryFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude files that matches the pattern. - */ - fileFilterPattern?: FilterPattern; - /** - * Optional configuration to turn off fetching metadata for directories. - */ - includeDirectories?: boolean; - /** - * Optional configuration to turn off fetching metadata for files. - */ - includeFiles?: boolean; - /** - * Optional configuration to turn off fetching metadata for spreadsheets. - */ - includeSpreadsheets?: boolean; - /** - * Optional configuration to turn off fetching metadata for worksheets. - */ - includeWorksheets?: boolean; - /** - * Optional configuration to soft delete directories in OpenMetadata if the source - * directories are deleted. Also, if the directory is deleted, all the associated entities - * like files, spreadsheets, worksheets, lineage, etc., with that directory will be deleted - */ - markDeletedDirectories?: boolean; - /** - * Optional configuration to soft delete files in OpenMetadata if the source files are - * deleted. Also, if the file is deleted, all the associated entities like lineage, etc., - * with that file will be deleted - */ - markDeletedFiles?: boolean; - /** - * Optional configuration to soft delete spreadsheets in OpenMetadata if the source - * spreadsheets are deleted. Also, if the spreadsheet is deleted, all the associated - * entities like worksheets, lineage, etc., with that spreadsheet will be deleted - */ - markDeletedSpreadsheets?: boolean; - /** - * Optional configuration to soft delete worksheets in OpenMetadata if the source worksheets - * are deleted. Also, if the worksheet is deleted, all the associated entities like lineage, - * etc., with that worksheet will be deleted - */ - markDeletedWorksheets?: boolean; - /** - * Regex to only include/exclude spreadsheets that matches the pattern. - */ - spreadsheetFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude worksheets that matches the pattern. - */ - worksheetFilterPattern?: FilterPattern; - /** - * Enable the 'Include Index Template' toggle to manage the ingestion of index template data. - */ - includeIndexTemplate?: boolean; - /** - * Optional configuration to turn off fetching sample data for search index. - */ - includeSampleData?: boolean; - /** - * Optional configuration to soft delete search indexes in OpenMetadata if the source search - * indexes are deleted. Also, if the search index is deleted, all the associated entities - * like lineage, etc., with that search index will be deleted - */ - markDeletedSearchIndexes?: boolean; - /** - * No. of records of sample data we want to ingest. - */ - sampleSize?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Fully qualified name of the entity to be tested, if we're working with a basic suite. - */ - entityFullyQualifiedName?: string; - /** - * Service connections to be used for the logical test suite. - */ - serviceConnections?: ServiceConnections[]; - /** - * List of test cases to be executed on the entity. If null, all test cases will be executed. - */ - testCases?: string[]; - /** - * Maximum number of events entities in a batch (Default 1000). - */ - batchSize?: number; - /** - * Certificate path to be added in configuration. The path should be local in the Ingestion - * Container. - */ - caCerts?: string; - recreateIndex?: boolean; - /** - * Region name. Required when using AWS Credentials. - */ - regionName?: string; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Connection Timeout - */ - timeout?: number; - /** - * Indicates whether to use aws credentials when connecting to OpenSearch in AWS. - */ - useAwsCredentials?: boolean; - /** - * Indicates whether to use SSL when connecting to ElasticSearch. By default, we will ignore - * SSL settings. - */ - useSSL?: boolean; - /** - * Indicates whether to verify certificates when using SSL connection to ElasticSearch. - * Ignored by default. Is set to true, make sure to send the certificates in the property - * `CA Certificates`. - */ - verifyCerts?: boolean; - /** - * Custom OpenMetadata Classification name for dbt tags. - */ - dbtClassificationName?: string; - /** - * Available sources to fetch DBT catalog and manifest files. - */ - dbtConfigSource?: DBTConfigurationSource; - /** - * Optional configuration to update the description from DBT or not - */ - dbtUpdateDescriptions?: boolean; - /** - * Optional configuration to update the owners from DBT or not - */ - dbtUpdateOwners?: boolean; - /** - * Optional configuration to search across databases for tables or not - */ - searchAcrossDatabases?: boolean; - /** - * Regex to only fetch tags that matches the pattern. - */ - tagFilterPattern?: FilterPattern; - /** - * Application configuration - */ - appConfig?: any[] | boolean | number | null | CollateAIAppConfig | string; - /** - * Application private configuration - */ - appPrivateConfig?: PrivateConfig; - /** - * Source Python Class Name to run the application - */ - sourcePythonClass?: string; - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete api collections in OpenMetadata if the source - * collections are deleted. Also, if the collection is deleted, all the associated entities - * like endpoints, etc., with that collection will be deleted - */ - markDeletedApiCollections?: boolean; - /** - * Optional value of the ingestion runner name responsible for running the workflow - */ - ingestionRunner?: string; - /** - * List of operations to be performed on the service - */ - operations?: Operation[]; - /** - * Service to be modified - */ - service?: EntityReference; -} - -/** - * Configuration for the CollateAI External Application. - * - * Configuration for the Automator External Application. - * - * This schema defines the Slack App Token Configuration - * - * Configuration for the Collate AI Quality Agent. - * - * No configuration needed to instantiate the Data Insights Pipeline. The logic is handled - * in the backend. - * - * Search Indexing App. - * - * Cache Warmup Application Configuration. - * - * Configuration for the AutoPilot Application. - */ -export interface CollateAIAppConfig { - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - filter?: string; - /** - * Patch the description if it is empty, instead of raising a suggestion - * - * Patch the tier if it is empty, instead of raising a suggestion - */ - patchIfEmpty?: boolean; - /** - * Application Type - */ - type?: CollateAIAppConfigType; - /** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - */ - actions?: Action[]; - /** - * Entities selected to run the automation. - */ - resources?: Resource; - /** - * Bot Token - */ - botToken?: string; - /** - * User Token - */ - userToken?: string; - /** - * Whether the suggested tests should be active or not upon suggestion - * - * Whether the AutoPilot Workflow should be active or not. - */ - active?: boolean; - backfillConfiguration?: BackfillConfiguration; - /** - * Maximum number of events processed at a time (Default 100). - * - * Maximum number of events sent in a batch (Default 100). - * - * Number of entities to process in each batch. - */ - batchSize?: number; - moduleConfiguration?: ModuleConfiguration; - /** - * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property - * Type and are facing errors. Bear in mind that recreating the index will delete your - * DataAssets and a backfill will be needed. - */ - recreateDataAssetsIndex?: boolean; - sendToAdmins?: boolean; - sendToTeams?: boolean; - /** - * Enable automatic performance tuning based on cluster capabilities and database entity - * count - */ - autoTune?: boolean; - /** - * Number of threads to use for reindexing - * - * Number of parallel threads for processing entities and warming cache. - */ - consumerThreads?: number; - /** - * List of Entities to Reindex - * - * List of entity types to warm up in cache. Use 'all' to warm up all entity types. - */ - entities?: string[]; - /** - * Initial backoff time in milliseconds - */ - initialBackoff?: number; - /** - * Maximum backoff time in milliseconds - */ - maxBackoff?: number; - /** - * Maximum number of concurrent requests to the search index - */ - maxConcurrentRequests?: number; - /** - * Maximum number of retries for a failed request - */ - maxRetries?: number; - /** - * Maximum number of events sent in a batch (Default 100). - */ - payLoadSize?: number; - /** - * Number of threads to use for reindexing - */ - producerThreads?: number; - /** - * Queue Size to user internally for reindexing. - * - * Internal queue size for entity processing pipeline. - */ - queueSize?: number; - /** - * This schema publisher run modes. - */ - recreateIndex?: boolean; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Force cache warmup even if another instance is detected (use with caution). - */ - force?: boolean; - /** - * Enter the retention period for Activity Threads of type = 'Conversation' records in days - * (e.g., 30 for one month, 60 for two months). - */ - activityThreadsRetentionPeriod?: number; - /** - * Enter the retention period for change event records in days (e.g., 7 for one week, 30 for - * one month). - */ - changeEventRetentionPeriod?: number; - /** - * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two - * months). - */ - profileDataRetentionPeriod?: number; - /** - * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for - * two months). - */ - testCaseResultsRetentionPeriod?: number; - /** - * Service Entity Link for which to trigger the application. - */ - entityLink?: string; - [property: string]: any; -} - -/** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - * - * Apply Classification Tags to the selected assets. - * - * Remove Classification Tags Action Type - * - * Apply Glossary Terms to the selected assets. - * - * Remove Glossary Terms Action Type - * - * Add domains to the selected assets. - * - * Remove domains from the selected assets. - * - * Apply Tags to the selected assets. - * - * Add a Custom Property to the selected assets. - * - * Remove Owner Action Type - * - * Add an owner to the selected assets. - * - * Add Test Cases to the selected assets. - * - * Remove Test Cases Action Type - * - * Add owners to the selected assets. - * - * Remove Custom Properties Action Type - * - * Add a Data Product to the selected assets. - * - * Remove a Data Product to the selected assets. - * - * Propagate description, tags and glossary terms via lineage - * - * ML Tagging action configuration for external automator. - */ -export interface Action { - /** - * Apply tags to the children of the selected assets that match the criteria. E.g., columns, - * tasks, topic fields,... - * - * Remove tags from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply terms to the children of the selected assets that match the criteria. E.g., - * columns, tasks, topic fields,... - * - * Remove terms from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply the description to the children of the selected assets that match the criteria. - * E.g., columns, tasks, topic fields,... - * - * Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Add tests to the selected table columns - * - * Remove tests to the selected table columns - */ - applyToChildren?: string[]; - /** - * Update tags even if they are already defined in the asset. By default, incoming tags are - * merged with the existing ones. - * - * Update terms even if they are already defined in the asset. By default, incoming terms - * are merged with the existing ones. - * - * Update the domains even if they are defined in the asset. By default, we will only apply - * the domains to assets without domains. - * - * Update the description even if they are already defined in the asset. By default, we'll - * only add the descriptions to assets without the description set. - * - * Update the Custom Property even if it is defined in the asset. By default, we will only - * apply the owners to assets without the given Custom Property informed. - * - * Update the tier even if it is defined in the asset. By default, we will only apply the - * tier to assets without tier. - * - * Update the test even if it is defined in the asset. By default, we will only apply the - * test to assets without the existing test already existing. - * - * Update the owners even if it is defined in the asset. By default, we will only apply the - * owners to assets without owner. - * - * Update the Data Product even if the asset belongs to a different Domain. By default, we - * will only add the Data Product if the asset has no Domain, or it belongs to the same - * domain as the Data Product. - * - * Update descriptions, tags and Glossary Terms via lineage even if they are already defined - * in the asset. By default, descriptions are only updated if they are not already defined - * in the asset, and incoming tags are merged with the existing ones. - */ - overwriteMetadata?: boolean; - /** - * Classification Tags to apply (source must be 'Classification') - * - * Classification Tags to remove (source must be 'Classification') - */ - tags?: TierElement[]; - /** - * Application Type - */ - type: ActionType; - /** - * Remove tags from all the children and parent of the selected assets. - * - * Remove terms from all the children and parent of the selected assets. - * - * Remove descriptions from all the children and parent of the selected assets. - */ - applyToAll?: boolean; - /** - * Remove tags by its label type - * - * Remove terms by its label type - */ - labels?: LabelElement[]; - /** - * Glossary Terms to apply - * - * Glossary Terms to remove - */ - terms?: TierElement[]; - /** - * Domains to apply - */ - domains?: EntityReference[]; - /** - * Description to apply - */ - description?: string; - /** - * Owners to apply - * - * Custom Properties keys to remove - */ - customProperties?: any; - /** - * tier to apply - */ - tier?: TierElement; - /** - * Test Cases to apply - */ - testCases?: TestCaseDefinitions[]; - /** - * Remove all test cases - */ - removeAll?: boolean; - /** - * Test Cases to remove - */ - testCaseDefinitions?: string[]; - /** - * Owners to apply - */ - owners?: EntityReference[]; - /** - * Data Products to apply - * - * Data Products to remove - */ - dataProducts?: EntityReference[]; - /** - * Propagate the metadata to columns via column-level lineage. - */ - propagateColumnLevel?: boolean; - /** - * Propagate description through lineage - */ - propagateDescription?: boolean; - /** - * Propagate domains from the parent through lineage - */ - propagateDomains?: boolean; - /** - * Propagate glossary terms through lineage - */ - propagateGlossaryTerms?: boolean; - /** - * Propagate owner from the parent - */ - propagateOwner?: boolean; - /** - * Propagate the metadata to the parents (e.g., tables) via lineage. - */ - propagateParent?: boolean; - /** - * Propagate tags through lineage - */ - propagateTags?: boolean; - /** - * Propagate tier from the parent - */ - propagateTier?: boolean; - /** - * Number of levels to propagate lineage. If not set, it will propagate to all levels. - */ - propagationDepth?: number; - /** - * List of configurations to stop propagation based on conditions - */ - propagationStopConfigs?: PropagationStopConfig[]; -} - -/** - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Service to be modified - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Remove tags by its label type - * - * Remove terms by its label type - */ -export enum LabelElement { - Automated = "Automated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Configuration to stop lineage propagation based on conditions - */ -export interface PropagationStopConfig { - /** - * The metadata attribute to check for stopping propagation - */ - metadataAttribute: MetadataAttribute; - /** - * List of attribute values that will stop propagation when any of them is matched - */ - value: Array; -} - -/** - * The metadata attribute to check for stopping propagation - */ -export enum MetadataAttribute { - Description = "description", - Domains = "domains", - GlossaryTerms = "glossaryTerms", - Owner = "owner", - Tags = "tags", - Tier = "tier", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - * - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Service to be modified - */ -export interface TagLabel { - /** - * Description for the tag label. - * - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this tag. - * - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Link to the tag resource. - * - * Link to the entity resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType?: LabelTypeEnum; - /** - * Name of the tag or glossary term. - * - * Name of the entity instance. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source?: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state?: State; - style?: Style; - tagFQN?: string; - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id?: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type?: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelTypeEnum { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * Cover image configuration for the entity. - */ - coverImage?: CoverImage; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * Cover image configuration for the entity. - * - * Cover image configuration for an entity. This is used to display a banner or header image - * for entities like Domain, Glossary, Data Product, etc. - */ -export interface CoverImage { - /** - * Position of the cover image in CSS background-position format. Supports keywords (top, - * center, bottom) or pixel values (e.g., '20px 30px'). - */ - position?: string; - /** - * URL of the cover image. - */ - url?: string; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - */ -export interface TierElement { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelTypeEnum; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * An explanation of why this tag was proposed, specially for autoclassification tags - */ - reason?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Minimum set of requirements to get a Test Case request ready - */ -export interface TestCaseDefinitions { - /** - * Compute the passed and failed row count for the test case. - */ - computePassedFailedRowCount?: boolean; - parameterValues?: TestCaseParameterValue[]; - /** - * Tags to apply - */ - tags?: TierElement[]; - /** - * Fully qualified name of the test definition. - */ - testDefinition?: string; - /** - * If the test definition supports it, use dynamic assertion to evaluate the test case. - */ - useDynamicAssertion?: boolean; - [property: string]: any; -} - -/** - * This schema defines the parameter values that can be passed for a Test Case. - */ -export interface TestCaseParameterValue { - /** - * name of the parameter. Must match the parameter names in testCaseParameterDefinition - */ - name?: string; - /** - * value to be passed for the Parameters. These are input from Users. We capture this in - * string and convert during the runtime. - */ - value?: string; - [property: string]: any; -} - -/** - * Application Type - * - * Add Tags action type. - * - * Remove Classification Tags Action Type. - * - * Add Terms action type. - * - * Remove Terms Action Type. - * - * Add Domain Action Type. - * - * Remove Domain Action Type - * - * Add Description Action Type. - * - * Add Custom Properties Action Type. - * - * Remove Description Action Type - * - * Add Tier Action Type. - * - * Remove Tier Action Type - * - * Add Test Case Action Type. - * - * Remove Test Case Action Type - * - * Add Owner Action Type. - * - * Remove Owner Action Type - * - * Remove Custom Properties Action Type. - * - * Add Data Products Action Type. - * - * Remove Data Products Action Type. - * - * Lineage propagation action type. - * - * ML PII Tagging action type. - */ -export enum ActionType { - AddCustomPropertiesAction = "AddCustomPropertiesAction", - AddDataProductAction = "AddDataProductAction", - AddDescriptionAction = "AddDescriptionAction", - AddDomainAction = "AddDomainAction", - AddOwnerAction = "AddOwnerAction", - AddTagsAction = "AddTagsAction", - AddTermsAction = "AddTermsAction", - AddTestCaseAction = "AddTestCaseAction", - AddTierAction = "AddTierAction", - LineagePropagationAction = "LineagePropagationAction", - MLTaggingAction = "MLTaggingAction", - RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", - RemoveDataProductAction = "RemoveDataProductAction", - RemoveDescriptionAction = "RemoveDescriptionAction", - RemoveDomainAction = "RemoveDomainAction", - RemoveOwnerAction = "RemoveOwnerAction", - RemoveTagsAction = "RemoveTagsAction", - RemoveTermsAction = "RemoveTermsAction", - RemoveTestCaseAction = "RemoveTestCaseAction", - RemoveTierAction = "RemoveTierAction", -} - -/** - * Backfill Configuration - */ -export interface BackfillConfiguration { - /** - * Enable Backfill for the configured dates - */ - enabled?: boolean; - /** - * Date for which the backfill will end - */ - endDate?: Date; - /** - * Date from which to start the backfill - */ - startDate?: Date; - [property: string]: any; -} - -/** - * Different Module Configurations - */ -export interface ModuleConfiguration { - /** - * App Analytics Module configuration - */ - appAnalytics: AppAnalyticsConfig; - /** - * Cost Analysis Insights Module configuration - */ - costAnalysis: CostAnalysisConfig; - /** - * Data Assets Insights Module configuration - */ - dataAssets: DataAssetsConfig; - /** - * Data Quality Insights Module configuration - */ - dataQuality: DataQualityConfig; -} - -/** - * App Analytics Module configuration - */ -export interface AppAnalyticsConfig { - /** - * If Enabled, App Analytics insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Cost Analysis Insights Module configuration - */ -export interface CostAnalysisConfig { - /** - * If Enabled, Cost Analysis insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Data Assets Insights Module configuration - */ -export interface DataAssetsConfig { - /** - * If Enabled, Data Asset insights will be populated when the App runs. - */ - enabled: boolean; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Defines the number of days the Data Assets Insights information will be kept. After it - * they will be deleted. - */ - retention?: number; - serviceFilter?: ServiceFilter; -} - -export interface ServiceFilter { - serviceName?: string; - serviceType?: string; -} - -/** - * Data Quality Insights Module configuration - */ -export interface DataQualityConfig { - /** - * If Enabled, Data Quality insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Entities selected to run the automation. - */ -export interface Resource { - /** - * Filter JSON tree to be used for rendering the filters in the UI. This comes from - * Immutable Tree type of react-awesome-query-builder. - */ - filterJsonTree?: string; - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - queryFilter?: string; - /** - * Type of the entity. E.g., 'table', 'chart',... - */ - type?: string[]; - [property: string]: any; -} - -/** - * Application Type - * - * Application type. - */ -export enum CollateAIAppConfigType { - AutoPilotApplication = "AutoPilotApplication", - Automator = "Automator", - CacheWarmup = "CacheWarmup", - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - CollateAITierAgent = "CollateAITierAgent", - DataInsights = "DataInsights", - DataInsightsReport = "DataInsightsReport", - SearchIndexing = "SearchIndexing", -} - -/** - * Application private configuration - * - * Private Configuration for the CollateAI External Application. - */ -export interface PrivateConfig { - /** - * Collate Server public URL. WAII will use this information to interact with the server. - * E.g., https://sandbox.getcollate.io - */ - collateURL?: string; - /** - * Limits for the CollateAI Application. - */ - limits?: AppLimitsConfig; - /** - * WAII API Token - */ - token?: string; - /** - * WAII API host URL - */ - waiiInstance?: string; - [property: string]: any; -} - -/** - * Limits for the CollateAI Application. - * - * Private Configuration for the App Limits. - */ -export interface AppLimitsConfig { - /** - * The records of the limits. - */ - actions: { [key: string]: number }; - /** - * The start of this limit cycle. - */ - billingCycleStart: Date; -} - -/** - * Available sources to fetch DBT catalog and manifest files. - * - * dbt Cloud configuration. - * - * DBT Catalog, Manifest and Run Results file path config. - * - * DBT Catalog, Manifest and Run Results HTTP path configuration. - * - * DBT Catalog, Manifest and Run Results files in S3 bucket. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in GCS storage. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in Azure bucket. We will search for - * catalog.json, manifest.json and run_results.json. - */ -export interface DBTConfigurationSource { - /** - * dbt cloud account Id - */ - dbtCloudAccountId?: string; - /** - * dbt cloud account authentication token - */ - dbtCloudAuthToken?: string; - /** - * dbt cloud job id. - */ - dbtCloudJobId?: string; - /** - * In case of multiple projects in a dbt cloud account, specify the project's id from which - * you want to extract the dbt run artifacts - */ - dbtCloudProjectId?: string; - /** - * URL to connect to your dbt cloud instance. E.g., https://cloud.getdbt.com or - * https://emea.dbt.com/ - */ - dbtCloudUrl?: string; - /** - * dbt Configuration type - */ - dbtConfigType: DbtConfigType; - /** - * DBT catalog file path to extract dbt models with their column schemas. - */ - dbtCatalogFilePath?: string; - /** - * DBT manifest file path to extract dbt models and associate with tables. - */ - dbtManifestFilePath?: string; - /** - * DBT run results file path to extract the test results information. - */ - dbtRunResultsFilePath?: string; - /** - * DBT sources file path to extract the freshness test result. - */ - dbtSourcesFilePath?: string; - /** - * DBT catalog http file path to extract dbt models with their column schemas. - */ - dbtCatalogHttpPath?: string; - /** - * DBT manifest http file path to extract dbt models and associate with tables. - */ - dbtManifestHttpPath?: string; - /** - * DBT run results http file path to extract the test results information. - */ - dbtRunResultsHttpPath?: string; - /** - * DBT sources http file path to extract freshness test results information. - */ - dbtSourcesHttpPath?: string; - /** - * Details of the bucket where the dbt files are stored - */ - dbtPrefixConfig?: DBTPrefixConfig; - dbtSecurityConfig?: Credentials; -} - -/** - * dbt Configuration type - */ -export enum DbtConfigType { - Azure = "azure", - Cloud = "cloud", - Gcs = "gcs", - HTTP = "http", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the dbt files are stored - */ -export interface DBTPrefixConfig { - /** - * Name of the bucket where the dbt files are stored - */ - dbtBucketName?: string; - /** - * Path of the folder where the dbt files are stored - */ - dbtObjectPrefix?: string; -} - -/** - * Use incremental Metadata extraction after the first execution. This is commonly done by - * getting the changes from Audit tables on the supporting databases. - */ -export interface IncrementalMetadataExtractionConfiguration { - /** - * If True, enables Metadata Extraction to be incremental - */ - enabled: boolean; - /** - * Number os days to search back for a successful pipeline run. The timestamp of the last - * found successful pipeline run will be used as a base to search for updated entities. - */ - lookbackDays?: number; - /** - * Number of days to add to the last successful pipeline run timestamp to search for updated - * entities. - */ - safetyMarginDays?: number; -} - -/** - * Details required to generate Lineage - */ -export interface LineageInformation { - /** - * List of service path prefixes for lineage matching. Supported formats: DBServiceName, - * DBServiceName.DatabaseName, DBServiceName.DatabaseName.SchemaName, or - * DBServiceName.DatabaseName.SchemaName.TableName - */ - dbServicePrefixes?: string[]; - /** - * List of Database Service Names for creation of lineage - */ - dbServiceNames?: string[]; - /** - * List of Storage Service Names for creation of lineage - */ - storageServiceNames?: string[]; - [property: string]: any; -} - -/** - * Operation to be performed on the entity - */ -export interface Operation { - /** - * Entity to be modified - */ - entityLink: string; - /** - * The id of the operation - */ - id: string; - /** - * The configuration for the operation to be applied - */ - parameters: ReverseIngestionConfig; - /** - * Templated SQL command to be used for the operation. Context parameters will be populated - * based on the event type. - */ - SQLTemplate?: string; - /** - * Type of operation to perform - */ - type: OperationType; -} - -/** - * The configuration for the operation to be applied - * - * Configuration for updating descriptions - * - * Configuration for updating owners - * - * Configuration for updating tags - */ -export interface ReverseIngestionConfig { - /** - * New description of the service - */ - newDescription?: string; - /** - * Previous description of the service - */ - previousDescription?: string; - /** - * Added owners to be applied - */ - addedOwners?: EntityReference[]; - /** - * Removed owners from the entity - */ - removedOwners?: EntityReference[]; - /** - * Added tags to be applied - */ - addedTags?: TierElement[]; - /** - * Removed tags of the entity - */ - removedTags?: TierElement[]; -} - -/** - * Type of operation to perform - */ -export enum OperationType { - UpdateDescription = "UPDATE_DESCRIPTION", - UpdateOwner = "UPDATE_OWNER", - UpdateTags = "UPDATE_TAGS", -} - -/** - * Advanced configuration for managing owners at multiple hierarchy levels (Service, - * Database, Schema, Table) with custom mappings and inheritance rules. - * - * Configuration for assigning owners to ingested entities following topology hierarchy with - * inheritance support - */ -export interface OwnerConfiguration { - /** - * Owner for database entities. Can be a single owner for all databases, or a map of - * database names to owner(s). - */ - database?: { [key: string]: string[] | string } | string; - /** - * Owner for schema entities. Can be a single owner for all schemas, or a map of schema FQNs - * to owner(s). - */ - databaseSchema?: { [key: string]: string[] | string } | string; - /** - * Default owner applied to all entities when no specific owner is configured (user or team - * name/email) - */ - default?: string; - /** - * Enable child entities to inherit owner from parent entities when they don't have a - * specific owner configured - */ - enableInheritance?: boolean; - /** - * Owner for the service level - */ - service?: string; - /** - * Owner for table entities. Can be a single owner for all tables, or a map of table FQNs to - * owner(s). - */ - table?: { [key: string]: string[] | string } | string; -} - -/** - * Processing Engine Configuration. If not provided, the Native Engine will be used by - * default. - * - * Configuration for the native metadata ingestion engine - * - * This schema defines the configuration for a Spark Engine runner. - */ -export interface ProcessingEngine { - /** - * The type of the engine configuration - */ - type: ProcessingEngineType; - config?: Config; - /** - * Spark Connect Remote URL. - */ - remote?: string; -} - -export interface Config { - /** - * Additional Spark configuration properties as key-value pairs. - */ - extraConfig?: { [key: string]: any }; - /** - * Temporary path to store the data. - */ - tempPath?: string; - [property: string]: any; -} - -/** - * The type of the engine configuration - */ -export enum ProcessingEngineType { - Native = "Native", - Spark = "Spark", -} - -/** - * Type of Profile Sample (percentage or rows) - */ -export enum ProfileSampleType { - Percentage = "PERCENTAGE", - Rows = "ROWS", -} - -/** - * Type of Sampling Method (BERNOULLI or SYSTEM) - */ -export enum SamplingMethodType { - Bernoulli = "BERNOULLI", - System = "SYSTEM", -} - -/** - * Service connections available for the logical test suite. - */ -export interface ServiceConnections { - /** - * Connection configuration for the source. ex: mysql , tableau connection. - */ - serviceConnection: ServiceConnection; - serviceName: string; -} - -/** - * No manifest file available. Ingestion would look for bucket-level metadata file instead - * - * Storage Metadata Manifest file path config. - * - * Storage Metadata Manifest file HTTP path config. - * - * Storage Metadata Manifest file S3 path config. - * - * Storage Metadata Manifest file ADLS path config. - * - * Storage Metadata Manifest file GCS path config. - */ -export interface StorageMetadataConfigurationSource { - /** - * Storage Metadata manifest file path to extract locations to ingest from. - */ - manifestFilePath?: string; - /** - * Storage Metadata manifest http file path to extract locations to ingest from. - */ - manifestHttpPath?: string; - prefixConfig?: StorageMetadataBucketDetails; - securityConfig?: Credentials; -} - -/** - * Details of the bucket where the storage metadata manifest file is stored - */ -export interface StorageMetadataBucketDetails { - /** - * Name of the top level container where the storage metadata file is stored - */ - containerName: string; - /** - * Path of the folder where the storage metadata file is stored. If the file is at the root, - * you can keep it empty. - */ - objectPrefix?: string; -} - -/** - * Pipeline type - * - * Database Source Config Metadata Pipeline type - * - * Database Source Config Usage Pipeline type - * - * Dashboard Source Config Metadata Pipeline type - * - * Messaging Source Config Metadata Pipeline type - * - * Profiler Source Config Pipeline type - * - * Pipeline Source Config Metadata Pipeline type - * - * MlModel Source Config Metadata Pipeline type - * - * Object Store Source Config Metadata Pipeline type - * - * Drive Source Config Metadata Pipeline type - * - * Search Source Config Metadata Pipeline type - * - * DBT Config Pipeline type - * - * Pipeline Source Config For Application Pipeline type. Nothing is required. - * - * Api Source Config Metadata Pipeline type - * - * Reverse Ingestion Config Pipeline type - */ -export enum FluffyType { - APIMetadata = "ApiMetadata", - Application = "Application", - AutoClassification = "AutoClassification", - DashboardMetadata = "DashboardMetadata", - DataInsight = "dataInsight", - DatabaseLineage = "DatabaseLineage", - DatabaseMetadata = "DatabaseMetadata", - DatabaseUsage = "DatabaseUsage", - Dbt = "DBT", - DriveMetadata = "DriveMetadata", - MessagingMetadata = "MessagingMetadata", - MetadataToElasticSearch = "MetadataToElasticSearch", - MlModelMetadata = "MlModelMetadata", - PipelineMetadata = "PipelineMetadata", - Profiler = "Profiler", - ReverseIngestion = "ReverseIngestion", - SearchMetadata = "SearchMetadata", - StorageMetadata = "StorageMetadata", - TestSuite = "TestSuite", -} - -/** - * Configuration for Stage Component in the OpenMetadata Ingestion Framework. - */ -export interface Stage { - config?: { [key: string]: any }; - /** - * Type of stage component ex: table-usage - */ - type: string; -} - -/** - * Configuration for the entire Ingestion Workflow. - */ -export interface WorkflowConfig { - config?: { [key: string]: any }; - loggerLevel?: LogLevels; - openMetadataServerConfig: OpenMetadataConnection; - /** - * Control if we want to flag the workflow as failed if we encounter any processing errors. - */ - raiseOnError?: boolean; - /** - * The percentage of successfully processed records that must be achieved for the pipeline - * to be considered successful. Otherwise, the pipeline will be marked as failed. - */ - successThreshold?: number; -} - -/** - * Supported logging levels - */ -export enum LogLevels { - Debug = "DEBUG", - Error = "ERROR", - Info = "INFO", - Warn = "WARN", -} - -/** - * OpenMetadata Connection Config - */ -export interface OpenMetadataConnection { - /** - * OpenMetadata server API version to use. - */ - apiVersion?: string; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: OpenMetadataServerConfigElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - */ - hostPort: string; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: ConsumerConfigSSLClass; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: OpenmetadataType; - /** - * Flag to verify SSL Certificate for OpenMetadata Server. - */ - verifySSL?: VerifySSL; -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface OpenMetadataServerConfigElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * Service Type - * - * OpenMetadata service type - */ -export enum OpenmetadataType { - OpenMetadata = "OpenMetadata", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts b/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts index eb4a151c86e..b4405060973 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts @@ -397,6 +397,10 @@ export interface PipelineServiceClientConfiguration { * OpenMetadata Server Endpoint */ openMetadataUrl?: string; + /** + * Bot Token + */ + botToken?: string; /** * Client Secret of the Application. */ @@ -405,7 +409,11 @@ export interface PipelineServiceClientConfiguration { * Signing Secret of the Application. Confirm that each request comes from Slack by * verifying its unique signature. */ - signingSecret?: string; + signingSecret?: string; + /** + * User Token + */ + userToken?: string; metricConfiguration?: MetricConfigurationDefinition[]; /** * Configurations of allowed searchable fields for each entity type