From 88fc3c65fbd5d549239479e725c06b26bc7d50b6 Mon Sep 17 00:00:00 2001 From: Keshav Mohta Date: Thu, 25 Sep 2025 16:04:18 +0530 Subject: [PATCH] feat: added oauth and azure ad for unity catalog --- .../mysql/postDataMigrationSQLScript.sql | 14 + .../postgres/postDataMigrationSQLScript.sql | 11 + .../mysql/postDataMigrationSQLScript.sql | 15 - .../postgres/postDataMigrationSQLScript.sql | 12 - .../source/database/databricks/client.py | 9 +- .../source/database/unitycatalog/client.py | 34 + .../database/unitycatalog/connection.py | 89 +- .../sqlalchemy/unitycatalog/sampler.py | 21 - .../converter/ClassConverterFactory.java | 2 + .../UnityCatalogConnectionClassConverter.java | 43 + .../database/unityCatalogConnection.json | 45 +- .../api/automations/createWorkflow.ts | 4559 ------------ .../api/services/createDatabaseService.ts | 2285 ------ .../createIngestionPipeline.ts | 6154 --------------- .../automations/testServiceConnection.ts | 4153 ----------- .../generated/entity/automations/workflow.ts | 4908 ------------ .../src/generated/entity/data/spreadsheet.ts | 1 + .../database/unityCatalogConnection.ts | 191 - .../services/connections/serviceConnection.ts | 4115 ----------- .../entity/services/databaseService.ts | 2463 ------- .../ingestionPipelines/ingestionPipeline.ts | 6568 ----------------- .../metadataIngestion/testSuitePipeline.ts | 4168 ----------- .../generated/metadataIngestion/workflow.ts | 6289 ---------------- 23 files changed, 232 insertions(+), 45917 deletions(-) create mode 100644 bootstrap/sql/migrations/native/1.10.0/mysql/postDataMigrationSQLScript.sql create mode 100644 bootstrap/sql/migrations/native/1.10.0/postgres/postDataMigrationSQLScript.sql create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/UnityCatalogConnectionClassConverter.java delete mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts delete mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts delete mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts delete mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts delete mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts delete mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/unityCatalogConnection.ts delete mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts delete mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts delete mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts delete mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts delete mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts diff --git a/bootstrap/sql/migrations/native/1.10.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.10.0/mysql/postDataMigrationSQLScript.sql new file mode 100644 index 00000000000..aa465519570 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.10.0/mysql/postDataMigrationSQLScript.sql @@ -0,0 +1,14 @@ +-- Migration script to restructure Databricks connection configuration +-- Move 'token' field from connection.config.token to connection.config.authType.token +UPDATE dbservice_entity +SET + json = JSON_SET ( + JSON_REMOVE (json, '$.connection.config.token'), + '$.connection.config.authType', + JSON_OBJECT ( + 'token', + JSON_EXTRACT (json, '$.connection.config.token') + ) + ) +WHERE + serviceType in ('Databricks', 'UnityCatalog'); \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.10.0/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.10.0/postgres/postDataMigrationSQLScript.sql new file mode 100644 index 00000000000..24d27c8ed96 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.10.0/postgres/postDataMigrationSQLScript.sql @@ -0,0 +1,11 @@ +-- Migration script to restructure Databricks connection configuration +-- Move 'token' field from connection.config.token to connection.config.authType.token + +UPDATE dbservice_entity +SET json = jsonb_set( + json #- '{connection,config,token}', + '{connection,config,authType}', + jsonb_build_object('token', json #> '{connection,config,token}'), + true +) +WHERE serviceType in ('Databricks', 'UnityCatalog'); diff --git a/bootstrap/sql/migrations/native/1.9.10/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.9.10/mysql/postDataMigrationSQLScript.sql index 162593aea78..8c857213a5b 100644 --- a/bootstrap/sql/migrations/native/1.9.10/mysql/postDataMigrationSQLScript.sql +++ b/bootstrap/sql/migrations/native/1.9.10/mysql/postDataMigrationSQLScript.sql @@ -5,18 +5,3 @@ UPDATE profiler_data_time_series SET json = JSON_SET(json, '$.profileData', json->'$.profileData.profileData') WHERE json->>'$.profileData.profileData' IS NOT NULL; - --- Migration script to restructure Databricks connection configuration --- Move 'token' field from connection.config.token to connection.config.authType.token -UPDATE dbservice_entity -SET - json = JSON_SET ( - JSON_REMOVE (json, '$.connection.config.token'), - '$.connection.config.authType', - JSON_OBJECT ( - 'token', - JSON_EXTRACT (json, '$.connection.config.token') - ) - ) -WHERE - serviceType = 'Databricks'; diff --git a/bootstrap/sql/migrations/native/1.9.10/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.9.10/postgres/postDataMigrationSQLScript.sql index f9714095890..b39f7e68b66 100644 --- a/bootstrap/sql/migrations/native/1.9.10/postgres/postDataMigrationSQLScript.sql +++ b/bootstrap/sql/migrations/native/1.9.10/postgres/postDataMigrationSQLScript.sql @@ -5,15 +5,3 @@ UPDATE profiler_data_time_series SET json = jsonb_set(json::jsonb, '{profileData}', json::jsonb->'profileData'->'profileData')::json WHERE json->'profileData'->>'profileData' IS NOT NULL; - --- Migration script to restructure Databricks connection configuration --- Move 'token' field from connection.config.token to connection.config.authType.token - -UPDATE dbservice_entity -SET json = jsonb_set( - json #- '{connection,config,token}', - '{connection,config,authType}', - jsonb_build_object('token', json #> '{connection,config,token}'), - true -) -WHERE serviceType = 'Databricks'; diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/client.py b/ingestion/src/metadata/ingestion/source/database/databricks/client.py index 2462de40ddd..8093c421544 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/client.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/client.py @@ -61,14 +61,13 @@ class DatabricksClient: ): self.config = config base_url, *_ = self.config.hostPort.split(":") - auth_token = self.config.token.get_secret_value() self.base_url = f"https://{base_url}{API_VERSION}" self.base_query_url = f"{self.base_url}{QUERIES_PATH}" self.base_job_url = f"https://{base_url}{JOB_API_VERSION}/jobs" self.jobs_list_url = f"{self.base_job_url}/list" self.jobs_run_list_url = f"{self.base_job_url}/runs/list" self.headers = { - "Authorization": f"Bearer {auth_token}", + **self._get_auth_header(), "Content-Type": "application/json", } self.api_timeout = self.config.connectionTimeout or 120 @@ -79,6 +78,12 @@ class DatabricksClient: self.engine = engine self.client = requests + def _get_auth_header(self) -> str: + """ + Method to get auth header + """ + return {"Authorization": f"Bearer {self.config.token.get_secret_value()}"} + def test_query_api_access(self) -> None: res = self.client.get( self.base_query_url, headers=self.headers, timeout=self.api_timeout diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/client.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/client.py index b06bb54e0d6..eed6a8d8bb4 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/client.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/client.py @@ -16,10 +16,24 @@ import traceback from requests import HTTPError +from metadata.generated.schema.entity.services.connections.database.databricks.azureAdSetup import ( + AzureAdSetup, +) +from metadata.generated.schema.entity.services.connections.database.databricks.databricksOAuth import ( + DatabricksOauth, +) +from metadata.generated.schema.entity.services.connections.database.databricks.personalAccessToken import ( + PersonalAccessToken, +) from metadata.ingestion.source.database.databricks.client import ( API_TIMEOUT, DatabricksClient, ) +from metadata.ingestion.source.database.unitycatalog.connection import ( + get_azure_ad_auth, + get_databricks_oauth_auth, + get_personal_access_token_auth, +) from metadata.ingestion.source.database.unitycatalog.models import ( LineageColumnStreams, LineageTableStreams, @@ -37,6 +51,26 @@ class UnityCatalogClient(DatabricksClient): UnityCatalogClient creates a Databricks connection based on DatabricksCredentials. """ + def _get_auth_header(self) -> str: + """ + Method to get auth header + """ + auth_method = { + PersonalAccessToken: get_personal_access_token_auth, + DatabricksOauth: get_databricks_oauth_auth, + AzureAdSetup: get_azure_ad_auth, + }.get(type(self.config.authType)) + if not auth_method: + raise ValueError( + f"Unsupported authentication type: {type(self.config.authType)}" + ) + + auth_args = auth_method(self.config) + if auth_args.get("access_token"): + return {"Authorization": f"Bearer {auth_args['access_token']}"} + + return {"Authorization": auth_args["credentials_provider"]()} + def get_table_lineage(self, table_name: str) -> LineageTableStreams: """ Method returns table lineage details diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py index a5e82bc9c45..f70fdf07778 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py @@ -12,16 +12,27 @@ """ Source connection handler """ +from copy import deepcopy from functools import partial from typing import Optional from databricks.sdk import WorkspaceClient +from databricks.sdk.core import Config, azure_service_principal, oauth_service_principal from sqlalchemy.engine import Engine from sqlalchemy.exc import DatabaseError from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) +from metadata.generated.schema.entity.services.connections.database.databricks.azureAdSetup import ( + AzureAdSetup, +) +from metadata.generated.schema.entity.services.connections.database.databricks.databricksOAuth import ( + DatabricksOauth, +) +from metadata.generated.schema.entity.services.connections.database.databricks.personalAccessToken import ( + PersonalAccessToken, +) from metadata.generated.schema.entity.services.connections.database.unityCatalogConnection import ( UnityCatalogConnection, ) @@ -50,8 +61,50 @@ from metadata.utils.logger import ingestion_logger logger = ingestion_logger() +def get_personal_access_token_auth(connection: UnityCatalogConnection) -> dict: + """ + Configure Personal Access Token authentication + """ + return {"access_token": connection.authType.token.get_secret_value()} + + +def get_databricks_oauth_auth(connection: UnityCatalogConnection): + """ + Create Databricks OAuth2 M2M credentials provider for Service Principal authentication + """ + + def credential_provider(): + hostname = connection.hostPort.split(":")[0] + config = Config( + host=f"https://{hostname}", + client_id=connection.authType.clientId, + client_secret=connection.authType.clientSecret.get_secret_value(), + ) + return oauth_service_principal(config) + + return {"credentials_provider": credential_provider} + + +def get_azure_ad_auth(connection: UnityCatalogConnection): + """ + Create Azure AD credentials provider for Azure Service Principal authentication + """ + + def credential_provider(): + hostname = connection.hostPort.split(":")[0] + config = Config( + host=f"https://{hostname}", + azure_client_secret=connection.authType.azureClientSecret.get_secret_value(), + azure_client_id=connection.authType.azureClientId, + azure_tenant_id=connection.authType.azureTenantId, + ) + return azure_service_principal(config) + + return {"credentials_provider": credential_provider} + + def get_connection_url(connection: UnityCatalogConnection) -> str: - url = f"{connection.scheme.value}://token:{connection.token.get_secret_value()}@{connection.hostPort}" + url = f"{connection.scheme.value}://{connection.hostPort}" return url @@ -59,10 +112,23 @@ def get_connection(connection: UnityCatalogConnection) -> WorkspaceClient: """ Create connection """ + client_params = {} + if isinstance(connection.authType, PersonalAccessToken): + client_params["token"] = connection.authType.token.get_secret_value() + elif isinstance(connection.authType, DatabricksOauth): + client_params["client_id"] = connection.authType.clientId + client_params[ + "client_secret" + ] = connection.authType.clientSecret.get_secret_value() + elif isinstance(connection.authType, AzureAdSetup): + client_params["azure_client_id"] = connection.authType.azureClientId + client_params[ + "azure_client_secret" + ] = connection.authType.azureClientSecret.get_secret_value() + client_params["azure_tenant_id"] = connection.authType.azureTenantId return WorkspaceClient( - host=get_host_from_host_port(connection.hostPort), - token=connection.token.get_secret_value(), + host=get_host_from_host_port(connection.hostPort), **client_params ) @@ -76,6 +142,23 @@ def get_sqlalchemy_connection(connection: UnityCatalogConnection) -> Engine: connection.connectionArguments = init_empty_connection_arguments() connection.connectionArguments.root["http_path"] = connection.httpPath + auth_method = { + PersonalAccessToken: get_personal_access_token_auth, + DatabricksOauth: get_databricks_oauth_auth, + AzureAdSetup: get_azure_ad_auth, + }.get(type(connection.authType)) + + if not auth_method: + raise ValueError( + f"Unsupported authentication type: {type(connection.authType)}" + ) + + auth_args = auth_method(connection) + + original_connection_arguments = connection.connectionArguments + connection.connectionArguments = deepcopy(original_connection_arguments) + connection.connectionArguments.root.update(auth_args) + return create_generic_db_connection( connection=connection, get_connection_url_fn=get_connection_url, diff --git a/ingestion/src/metadata/sampler/sqlalchemy/unitycatalog/sampler.py b/ingestion/src/metadata/sampler/sqlalchemy/unitycatalog/sampler.py index 7a977a0b628..eca11842280 100644 --- a/ingestion/src/metadata/sampler/sqlalchemy/unitycatalog/sampler.py +++ b/ingestion/src/metadata/sampler/sqlalchemy/unitycatalog/sampler.py @@ -15,9 +15,6 @@ supporting sqlalchemy abstraction layer """ -from metadata.generated.schema.entity.services.connections.database.databricksConnection import ( - DatabricksConnection, -) from metadata.sampler.sqlalchemy.databricks.sampler import DatabricksSamplerInterface @@ -27,22 +24,4 @@ class UnityCatalogSamplerInterface(DatabricksSamplerInterface): """ def __init__(self, *args, **kwargs): - # Convert Unity Catalog connection to Databricks and move token to authType. - kwargs["service_connection_config"] = DatabricksConnection.model_validate( - { - **( - ( - t := ( - cfg := kwargs["service_connection_config"].model_dump( - mode="json" - ) - ).pop("token") - ) - and cfg - ), - "type": "Databricks", - "authType": {"token": t}, - } - ) - super().__init__(*args, **kwargs) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java index 0b86f38c161..23ee42358ea 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java @@ -42,6 +42,7 @@ import org.openmetadata.schema.services.connections.database.RedshiftConnection; import org.openmetadata.schema.services.connections.database.SalesforceConnection; import org.openmetadata.schema.services.connections.database.SapHanaConnection; import org.openmetadata.schema.services.connections.database.TrinoConnection; +import org.openmetadata.schema.services.connections.database.UnityCatalogConnection; import org.openmetadata.schema.services.connections.database.datalake.GCSConfig; import org.openmetadata.schema.services.connections.database.deltalake.StorageConfig; import org.openmetadata.schema.services.connections.database.iceberg.IcebergFileSystem; @@ -106,6 +107,7 @@ public final class ClassConverterFactory { Map.entry(VertexAIConnection.class, new VertexAIConnectionClassConverter()), Map.entry(RangerConnection.class, new RangerConnectionClassConverter()), Map.entry(DatabricksConnection.class, new DatabricksConnectionClassConverter()), + Map.entry(UnityCatalogConnection.class, new UnityCatalogConnectionClassConverter()), Map.entry(CassandraConnection.class, new CassandraConnectionClassConverter()), Map.entry(SSISConnection.class, new SsisConnectionClassConverter()), Map.entry(WherescapeConnection.class, new WherescapeConnectionClassConverter())); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/UnityCatalogConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/UnityCatalogConnectionClassConverter.java new file mode 100644 index 00000000000..c88b4853d3a --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/UnityCatalogConnectionClassConverter.java @@ -0,0 +1,43 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.services.connections.database.UnityCatalogConnection; +import org.openmetadata.schema.services.connections.database.databricks.AzureADSetup; +import org.openmetadata.schema.services.connections.database.databricks.DatabricksOAuth; +import org.openmetadata.schema.services.connections.database.databricks.PersonalAccessToken; +import org.openmetadata.schema.utils.JsonUtils; + +/** Converter class to get a `UnityCatalogConnection` object. */ +public class UnityCatalogConnectionClassConverter extends ClassConverter { + + private static final List> CONFIG_SOURCE_CLASSES = + List.of(PersonalAccessToken.class, DatabricksOAuth.class, AzureADSetup.class); + + public UnityCatalogConnectionClassConverter() { + super(UnityCatalogConnection.class); + } + + @Override + public Object convert(Object object) { + UnityCatalogConnection unityCatalogConnection = + (UnityCatalogConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvert(unityCatalogConnection.getAuthType(), CONFIG_SOURCE_CLASSES) + .ifPresent(unityCatalogConnection::setAuthType); + + return unityCatalogConnection; + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/unityCatalogConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/unityCatalogConnection.json index 1deefea27b5..f27e42293bd 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/unityCatalogConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/unityCatalogConnection.json @@ -9,13 +9,17 @@ "databricksType": { "description": "Service type.", "type": "string", - "enum": ["UnityCatalog"], + "enum": [ + "UnityCatalog" + ], "default": "UnityCatalog" }, "databricksScheme": { "description": "SQLAlchemy driver scheme options.", "type": "string", - "enum": ["databricks+connector"], + "enum": [ + "databricks+connector" + ], "default": "databricks+connector" } }, @@ -37,11 +41,23 @@ "description": "Host and port of the Databricks service.", "type": "string" }, - "token": { - "title": "Token", - "description": "Generated Token to connect to Databricks.", - "type": "string", - "format": "password" + "authType": { + "title": "Authentication Type", + "description": "Choose between different authentication types for Databricks.", + "oneOf": [ + { + "title": "Personal Access Token", + "$ref": "./databricks/personalAccessToken.json" + }, + { + "title": "Databricks OAuth", + "$ref": "./databricks/databricksOAuth.json" + }, + { + "title": "Azure AD Setup", + "$ref": "./databricks/azureAdSetup.json" + } + ] }, "httpPath": { "title": "Http Path", @@ -78,7 +94,9 @@ "$ref": "../../../../type/filterPattern.json#/definitions/filterPattern", "default": { "includes": [], - "excludes": ["^information_schema$"] + "excludes": [ + "^information_schema$" + ] } }, "tableFilterPattern": { @@ -92,7 +110,9 @@ "$ref": "../../../../type/filterPattern.json#/definitions/filterPattern", "default": { "includes": [], - "excludes": ["^system$"] + "excludes": [ + "^system$" + ] } }, "supportsUsageExtraction": { @@ -126,5 +146,8 @@ } }, "additionalProperties": false, - "required": ["hostPort", "token"] -} + "required": [ + "hostPort", + "token" + ] +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts deleted file mode 100644 index 530cb731d7d..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts +++ /dev/null @@ -1,4559 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * A unit of work that will be triggered as an API call to the OpenMetadata server. - */ -export interface CreateWorkflow { - /** - * Description of the test connection def. - */ - description?: string; - /** - * Display Name that identifies this workflow definition. - */ - displayName?: string; - /** - * Fully qualified names of the domains the Workflow belongs to. - */ - domains?: string[]; - /** - * Name of the workflow. - */ - name: string; - /** - * Owners of this workflow. - */ - owners?: EntityReference[]; - /** - * Request body for a specific workflow type - */ - request: TestServiceConnectionRequest; - /** - * Response to the request. - */ - response?: TestConnectionResult; - /** - * Workflow computation status. - */ - status?: WorkflowStatus; - /** - * Type of the workflow. - */ - workflowType: WorkflowType; -} - -/** - * Owners of this workflow. - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Service to be modified - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Request body for a specific workflow type - * - * Test Service Connection to test user provided configuration is valid or not. - * - * Apply a set of operations on a service - */ -export interface TestServiceConnectionRequest { - /** - * Connection object. - */ - connection?: RequestConnection; - /** - * Type of the connection to test such as Snowflake, MySQL, Looker, etc. - */ - connectionType?: string; - /** - * Optional value of the ingestion runner name responsible for running the test - * - * Optional value of the ingestion runner name responsible for running the workflow - */ - ingestionRunner?: string; - /** - * Secrets Manager Provider to use for fetching secrets. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * Optional value that identifies this service name. - */ - serviceName?: string; - /** - * Type of service such as Database, Dashboard, Messaging, etc. - */ - serviceType?: ServiceType; - /** - * List of operations to be performed on the service - */ - operations?: Operation[]; - /** - * Service to be modified - */ - service?: EntityReference; - /** - * Pipeline type - */ - type?: ReverseIngestionType; -} - -/** - * Connection object. - * - * API Service Connection. - * - * Database Connection. - * - * Dashboard Connection. - * - * Pipeline Connection. - * - * MlModel Connection. - * - * Metadata Service Connection. - * - * Storage Connection. - * - * search Connection. - * - * Security Connection. - * - * Drive Connection. - */ -export interface RequestConnection { - config?: ConfigObject; -} - -/** - * REST Connection Config - * - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - * - * Looker Connection Config - * - * Metabase Connection Config - * - * PowerBI Connection Config - * - * PowerBIReportServer Connection Config - * - * Redash Connection Config - * - * Superset Connection Config - * - * Tableau Connection Config - * - * Mode Connection Config - * - * Custom Dashboard Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Domo Dashboard Connection Config - * - * QuickSight Connection Config - * - * Qlik Sense Connection Config - * - * Lightdash Connection Config - * - * MicroStrategy Connection Config - * - * Qlik Cloud Connection Config - * - * Sigma Connection Config - * - * ThoughtSpot Connection Config - * - * Grafana Connection Config - * - * Kafka Connection Config - * - * Redpanda Connection Config - * - * Kinesis Connection Config - * - * Custom Messaging Service Connection to build a source that is not supported by - * OpenMetadata yet. - * - * Airflow Metadata Database Connection Config - * - * Wherescape Metadata Database Connection Config - * - * SSIS Metadata Database Connection Config - * - * Glue Pipeline Connection Config - * - * Airbyte Metadata Database Connection Config - * - * Fivetran Metadata Database Connection Config - * - * Flink Metadata Connection Config - * - * Dagster Metadata Database Connection Config - * - * Nifi Metadata Pipeline Connection Config - * - * Domo Pipeline Connection Config - * - * Custom Pipeline Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Spline Metadata Database Connection Config - * - * Spark Metadata Pipeline Connection Config - * - * OpenLineage Connection Config - * - * KafkaConnect Connection Config - * - * DBTCloud Connection Config - * - * Matillion Connection - * - * Azure Data Factory Connection Config - * - * Stitch Connection - * - * Snowplow Pipeline Connection Config - * - * MlFlow Connection Config - * - * Sklearn Connection Config - * - * Custom MlModel Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * SageMaker Connection Config - * - * Google VertexAI Connection Config - * - * Amundsen Connection Config - * - * Metadata to ElasticSearch Connection Config - * - * OpenMetadata Connection Config - * - * Atlas Connection Config - * - * Alation Connection Config - * - * Alation Sink Connection Config - * - * S3 Connection. - * - * ADLS Connection. - * - * GCS Connection. - * - * Custom Storage Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * ElasticSearch Connection. - * - * OpenSearch Connection Config - * - * Custom Search Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * Apache Ranger Connection Config - * - * Google Drive Connection Config - * - * SharePoint Connection Config - * - * Custom Drive Connection to build a source that is not supported. - */ -export interface ConfigObject { - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Documentation URL for the schema. - */ - docURL?: string; - /** - * Open API Schema URL. - */ - openAPISchemaURL?: string; - /** - * Supports Metadata Extraction. - */ - supportsMetadataExtraction?: boolean; - /** - * Generated Token to connect to OpenAPI Schema. - * - * Generated Token to connect to Databricks. - * - * token to connect to Qlik Cloud. - * - * To Connect to Dagster Cloud - * - * Generated Token to connect to DBTCloud. - * - * Token to connect to Stitch api doc - */ - token?: string; - /** - * REST API Type - * - * Service Type - * - * Custom database service type - * - * Custom dashboard service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * OpenSearch Type - * - * Custom search service type - */ - type?: ConfigType; - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - * - * Azure Credentials - * - * GCP Credentials for Google Drive API - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - * - * URL to the Looker instance. - * - * Host and Port of the Metabase instance. - * - * Dashboard URL for PowerBI service. - * - * Dashboard URL for PowerBI Report Server. - * - * URL for the Redash instance - * - * URL for the superset instance. - * - * Tableau Server url. - * - * URL for the mode instance. - * - * URL for the Qlik instance. - * - * Address for your running Lightdash instance - * - * Host and Port of the MicroStrategy instance. - * - * Host and Port of the Qlik Cloud instance. - * - * Sigma API url. - * - * ThoughtSpot instance URL. Example: https://my-company.thoughtspot.cloud - * - * URL to the Grafana instance. - * - * Pipeline Service Management/UI URI. - * - * Pipeline Service Management/UI URL. - * - * Spline REST Server Host & Port. - * - * KafkaConnect Service Management/UI URI. - * - * Host and port of the Stitch API host - * - * Host and port of the Amundsen Neo4j Connection. This expect a URI format like: - * bolt://localhost:7687. - * - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - * - * Host and port of the Atlas service. - * - * Host and port of the Alation service. - * - * Host and port of the ElasticSearch service. - * - * Host and port of the OpenSearch service. - * - * Apache Ranger Admin URL. - */ - hostPort?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - awsConfig?: AWSCredentials; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to the Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - * - * Password to connect to Metabase. Required for basic authentication. - * - * Password to connect to PowerBI report server. - * - * Password to connect to MicroStrategy. - * - * Password to connect to Airbyte. - * - * password to connect to the Amundsen Neo4j Connection. - * - * password to connect to the Atlas. - */ - password?: string; - /** - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to the Salesforce. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - * - * Username to connect to Metabase. Required for basic authentication. - * - * Username to connect to PowerBI report server. - * - * Username for Redash - * - * Username to connect to MicroStrategy. This user should have privileges to read all the - * metadata in MicroStrategy. - * - * Username to connect to Airbyte. - * - * username to connect to the Amundsen Neo4j Connection. - * - * username to connect to the Atlas. This user should have privileges to read all the - * metadata in Atlas. - */ - username?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - * - * Types of methods used to authenticate to the tableau instance - * - * Types of methods used to authenticate to the alation instance - * - * Authentication type to connect to Apache Ranger. - */ - authType?: AuthenticationType | NoConfigAuthenticationTypes; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - * - * Connection timeout in seconds. - */ - connectionTimeout?: number | number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - * - * Available sources to fetch metadata. - */ - configSource?: DeltaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - */ - auth?: AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - * - * Snowplow BDP Organization ID - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Access token to connect to DOMO - * - * Access Token for Mode Dashboard - */ - accessToken?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * Client ID for DOMO - * - * Azure Application (client) ID for service principal authentication. - * - * User's Client ID. This user should have privileges to read all the metadata in Looker. - * - * client_id for PowerBI. - * - * client_id for Sigma. - * - * Application (client) ID from Azure Active Directory - */ - clientId?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - * - * Secret token to connect to DOMO - */ - secretToken?: string; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * Choose between Database connection or HDB User Store connection. - * - * Choose between API or database connection fetch metadata from superset. - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Matillion Auth Configuration - * - * Choose between mysql and postgres connection for alation database - */ - connection?: ConfigConnection; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * API key to authenticate with the SAP ERP APIs. - * - * API token to connect to Metabase. Use this instead of username/password for token-based - * authentication. - * - * API key of the redash instance to access. - * - * The personal access token you can generate in the Lightdash app under the user settings - * - * Service Account Token to authenticate to the Grafana APIs. Use Service Account Tokens - * (format: glsa_xxxx) for authentication. Legacy API Keys are no longer supported by - * Grafana as of January 2025. Both self-hosted and Grafana Cloud are supported. Requires - * Admin role for full metadata extraction. - * - * Fivetran API Secret. - * - * API Key for Snowplow Console API - */ - apiKey?: string; - /** - * Pagination limit used while querying the SAP ERP API for fetching the entities - * - * Pagination limit used while querying the tableau metadata API for getting data sources - * - * Pagination limit used for Alation APIs pagination - */ - paginationLimit?: number; - /** - * Boolean marking if we need to verify the SSL certs for Grafana. Default to True. - * - * Boolean marking if we need to verify the SSL certs for KafkaConnect REST API. True by - * default. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - */ - verifySSL?: boolean | VerifySSL; - /** - * Azure Application client secret for service principal authentication. - * - * User's Client Secret. - * - * clientSecret for PowerBI. - * - * clientSecret for Sigma. - * - * Application (client) secret from Azure Active Directory - */ - clientSecret?: string; - /** - * Azure Directory (tenant) ID for service principal authentication. - * - * Tenant ID for PowerBI. - * - * Directory (tenant) ID from Azure Active Directory - */ - tenantId?: string; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - */ - gitCredentials?: GitHubCredentials; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * Authority URI for the PowerBI service. - */ - authorityURI?: string; - /** - * Display Table Name from source instead of renamed table name for datamodel tables - */ - displayTableNameFromSource?: boolean; - /** - * Entity Limit set here will be used to paginate the PowerBi APIs - */ - pagination_entity_per_page?: number; - /** - * Source to get the .pbit files to extract lineage information - */ - pbitFilesSource?: PowerBIPbitFilesSource; - /** - * PowerBI secrets. - */ - scope?: string[]; - /** - * Fetch the PowerBI metadata using admin APIs - */ - useAdminApis?: boolean; - /** - * Web Portal Virtual Directory Name. - */ - webPortalVirtualDirectory?: string; - /** - * Version of the Redash instance - */ - redashVersion?: string; - /** - * Tableau API version. If not provided, the version will be used from the tableau server. - * - * Sigma API version. - * - * ThoughtSpot API version to use - * - * Airbyte API version. - * - * OpenMetadata server API version to use. - */ - apiVersion?: string; - /** - * Proxy URL for the tableau server. If not provided, the hostPort will be used. This is - * used to generate the dashboard & Chart URL. - */ - proxyURL?: string; - /** - * Tableau Site Name. - * - * SharePoint site name - */ - siteName?: string; - /** - * Access Token Password for Mode Dashboard - */ - accessTokenPassword?: string; - /** - * Filter query parameter for some of the Mode API calls - */ - filterQueryParam?: string; - /** - * Mode Workspace Name - */ - workspaceName?: string; - /** - * AWS Account ID - */ - awsAccountId?: string; - /** - * The authentication method that the user uses to sign in. - */ - identityType?: IdentityType; - /** - * The Amazon QuickSight namespace that contains the dashboard IDs in this request ( To be - * provided when identityType is `ANONYMOUS` ) - */ - namespace?: string; - certificates?: QlikCertificatesBy; - /** - * Qlik Sense Base URL, used for genrating dashboard & chat url - */ - displayUrl?: string; - /** - * User Directory. - */ - userDirectory?: string; - /** - * User ID. - */ - userId?: string; - /** - * Validate Host Name - */ - validateHostName?: boolean; - /** - * The Project UUID for your Lightdash instance - */ - projectUUID?: string; - /** - * Use if your Lightdash instance is behind a proxy like (Cloud IAP) - */ - proxyAuthentication?: string; - /** - * The Space UUID for your Lightdash instance - */ - spaceUUID?: string; - /** - * Login Mode for Microstrategy's REST API connection. You can authenticate with one of the - * following authentication modes: `Standard (1)`, `Anonymous (8)`. Default will be - * `Standard (1)`. If you're using demo account for Microstrategy, it will be needed to - * authenticate through loginMode `8`. - */ - loginMode?: string; - /** - * MicroStrategy Project Name - * - * Project name to create the refreshToken. Can be anything - */ - projectName?: string; - /** - * Space types of Qlik Cloud to filter the dashboards ingested into the platform. - */ - spaceTypes?: SpaceType[]; - /** - * ThoughtSpot authentication configuration - */ - authentication?: Authenticationation; - /** - * Org ID for multi-tenant ThoughtSpot instances. This is applicable for ThoughtSpot Cloud - * only. - */ - orgId?: string; - /** - * Page size for pagination in API requests. Default is 100. - */ - pageSize?: number; - /** - * basic.auth.user.info schema registry config property, Client HTTP credentials in the form - * of username:password. - */ - basicAuthUserInfo?: string; - /** - * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - * - * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - */ - bootstrapServers?: string; - /** - * Confluent Kafka Consumer Config. From - * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - * - * Confluent Redpanda Consumer Config - */ - consumerConfig?: { [key: string]: any }; - /** - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - */ - consumerConfigSSL?: ConsumerConfigSSLClass; - /** - * sasl.mechanism Consumer Config property - */ - saslMechanism?: SaslMechanismType; - /** - * sasl.password consumer config property - */ - saslPassword?: string; - /** - * sasl.username consumer config property - */ - saslUsername?: string; - /** - * Confluent Kafka Schema Registry Config. From - * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient - * - * Confluent Redpanda Schema Registry Config. - */ - schemaRegistryConfig?: { [key: string]: any }; - /** - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ - schemaRegistrySSL?: ConsumerConfigSSLClass; - /** - * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get - * topic schema from registry. - */ - schemaRegistryTopicSuffixName?: string; - /** - * Confluent Kafka Schema Registry URL. - * - * Confluent Redpanda Schema Registry URL. - */ - schemaRegistryURL?: string; - /** - * security.protocol consumer config property - * - * Kafka security protocol config - */ - securityProtocol?: KafkaSecurityProtocol; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Pipeline Service Number Of Status - */ - numberOfStatus?: number; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Underlying database connection - */ - databaseConnection?: DatabaseConnectionClass; - /** - * Underlying storage connection - */ - packageConnection?: S3Connection | string; - /** - * Fivetran API Secret. - */ - apiSecret?: string; - /** - * Fivetran API Limit For Pagination. - */ - limit?: number; - /** - * URL to the Dagster instance - * - * DBT cloud Access URL. - */ - host?: string; - /** - * Connection Time Limit Between OM and Dagster Graphql API in second - */ - timeout?: number; - /** - * We support username/password or client certificate authentication - */ - nifiConfig?: NifiCredentialsConfiguration; - /** - * Spline UI Host & Port. - */ - uiHostPort?: string; - /** - * service type of the messaging source - */ - brokersUrl?: string; - /** - * consumer group name - */ - consumerGroupName?: string; - /** - * initial Kafka consumer offset - */ - consumerOffsets?: InitialConsumerOffsets; - /** - * max allowed wait time - */ - poolTimeout?: number; - /** - * SASL Configuration details. - */ - saslConfig?: SASLClientConfig; - /** - * max allowed inactivity time - */ - sessionTimeout?: number; - /** - * topic from where Open lineage events will be pulled - */ - topicName?: string; - /** - * We support username/password or No Authentication - */ - KafkaConnectConfig?: UsernamePasswordAuthentication; - /** - * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. - * e.g. local_kafka - * - * service type of the messaging source - */ - messagingServiceName?: string[] | string; - /** - * ID of your DBT cloud account - */ - accountId?: string; - /** - * DBT cloud Metadata API URL. - */ - discoveryAPI?: string; - /** - * List of IDs of your DBT cloud jobs seperated by comma `,` - */ - jobIds?: string[]; - /** - * Number of runs to fetch from DBT cloud - */ - numberOfRuns?: number; - /** - * List of IDs of your DBT cloud projects seperated by comma `,` - */ - projectIds?: string[]; - /** - * The name of your azure data factory. - */ - factory_name?: string; - /** - * The name of your resource group the data factory is associated with. - */ - resource_group_name?: string; - /** - * Number of days in the past to filter pipeline runs. - */ - run_filter_days?: number; - /** - * The azure subscription identifier. - */ - subscription_id?: string; - /** - * Cloud provider where Snowplow is deployed - */ - cloudProvider?: CloudProvider; - /** - * Path to pipeline configuration files for Community deployment - */ - configPath?: string; - /** - * Snowplow Console URL for BDP deployment - */ - consoleUrl?: string; - /** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - */ - deployment?: SnowplowDeployment; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Mlflow Model registry backend. E.g., - * mysql+pymysql://mlflow:password@localhost:3307/experiments - */ - registryUri?: string; - /** - * Mlflow Experiment tracking URI. E.g., http://localhost:5000 - */ - trackingUri?: string; - /** - * location/region of google cloud project - */ - location?: string; - /** - * Enable encryption for the Amundsen Neo4j Connection. - */ - encrypted?: boolean; - /** - * Maximum connection lifetime for the Amundsen Neo4j Connection. - */ - maxConnectionLifeTime?: number; - /** - * Enable SSL validation for the Amundsen Neo4j Connection. - */ - validateSSL?: boolean; - /** - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - /** - * List of entities that you need to reindex - */ - entities?: string[]; - recreateIndex?: boolean; - runMode?: RunMode; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * service type of the data source. - */ - databaseServiceName?: string[]; - /** - * Name of the Entity Type available in Atlas. - */ - entity_type?: string; - /** - * Custom OpenMetadata Classification name for alation tags. - */ - alationTagClassificationName?: string; - /** - * Specifies if hidden datasources should be included while ingesting. - */ - includeHiddenDatasources?: boolean; - /** - * Specifies if undeployed datasources should be included while ingesting. - */ - includeUndeployedDatasources?: boolean; - /** - * Specifies if Dashboards are to be ingested while running the ingestion job. - */ - ingestDashboards?: boolean; - /** - * Specifies if Datasources are to be ingested while running the ingestion job. - */ - ingestDatasources?: boolean; - /** - * Specifies if Domains are to be ingested while running the ingestion job. - */ - ingestDomains?: boolean; - /** - * Specifies if Knowledge Articles are to be ingested while running the ingestion job. - */ - ingestKnowledgeArticles?: boolean; - /** - * Specifies if Users and Groups are to be ingested while running the ingestion job. - */ - ingestUsersAndGroups?: boolean; - datasourceLinks?: { [key: string]: string }; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Connection Timeout in Seconds - */ - connectionTimeoutSecs?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Email to impersonate using domain-wide delegation - */ - delegatedEmail?: string; - /** - * Specific shared drive ID to connect to - * - * SharePoint drive ID. If not provided, default document library will be used - */ - driveId?: string; - /** - * Extract metadata only for Google Sheets files - */ - includeGoogleSheets?: boolean; - /** - * Include shared/team drives in metadata extraction - */ - includeTeamDrives?: boolean; - /** - * SharePoint site URL - */ - siteUrl?: string; - [property: string]: any; -} - -/** - * We support username/password or No Authentication - * - * username/password auth - */ -export interface UsernamePasswordAuthentication { - /** - * KafkaConnect password to authenticate to the API. - */ - password?: string; - /** - * KafkaConnect user to authenticate to the API. - */ - username?: string; -} - -/** - * Regex to only fetch api collections with names matching the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - * - * Regex exclude or include charts that matches the pattern. - * - * Regex to exclude or include dashboards that matches the pattern. - * - * Regex exclude or include data models that matches the pattern. - * - * Regex to exclude or include projects that matches the pattern. - * - * Regex to only fetch topics that matches the pattern. - * - * Regex exclude pipelines. - * - * Regex to only fetch containers that matches the pattern. - * - * Regex to only fetch MlModels with names matching the pattern. - * - * Regex to only fetch search indexes that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - * - * Types of methods used to authenticate to the tableau instance - * - * Basic Auth Credentials - * - * Access Token Auth Credentials - * - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * API Access Token Auth Credentials - * - * Basic Auth Configuration for ElasticSearch - * - * SSL Certificates By Path - * - * AWS credentials configs. - * - * Authentication type to connect to Apache Ranger. - * - * Configuration for connecting to Ranger Basic Auth. - */ -export interface AuthenticationType { - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - /** - * Password to connect to source. - * - * Password to access the service. - * - * Elastic Search Password for Login - * - * Ranger password to authenticate to the API. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; - /** - * Username to access the service. - * - * Elastic Search Username for Login - * - * Ranger user to authenticate to the API. - */ - username?: string; - /** - * Personal Access Token Name. - */ - personalAccessTokenName?: string; - /** - * Personal Access Token Secret. - */ - personalAccessTokenSecret?: string; - /** - * Access Token for the API - */ - accessToken?: string; - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -/** - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * Basic Auth Credentials - * - * API Access Token Auth Credentials - */ -export interface Authenticationation { - /** - * Password to access the service. - */ - password?: string; - /** - * Username to access the service. - */ - username?: string; - /** - * Access Token for the API - */ - accessToken?: string; -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: Authentication; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum Authentication { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: Credentials | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Qlik Authentication Certificate By Values - * - * Qlik Authentication Certificate File Path - */ -export interface QlikCertificatesBy { - sslConfig?: ConsumerConfigSSLClass; - /** - * Client Certificate - */ - clientCertificate?: string; - /** - * Client Key Certificate. - */ - clientKeyCertificate?: string; - /** - * Root Certificate. - */ - rootCertificate?: string; - [property: string]: any; -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface ConsumerConfigSSLClass { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Cloud provider where Snowplow is deployed - */ -export enum CloudProvider { - Aws = "AWS", - Azure = "Azure", - Gcp = "GCP", -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface DeltaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConfigSourceConnection; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: SecurityConfigClass; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConfigSourceConnection { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - * - * GCP Credentials - * - * GCP credentials configs. - * - * GCP Credentials for Google Drive API - * - * AWS credentials configs. - */ -export interface SecurityConfigClass { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - * - * Choose between API or database connection fetch metadata from superset. - * - * Superset API Connection Config - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Lineage Backend Connection Config - * - * SQLite Database Connection Config - * - * Matillion Auth Configuration - * - * Matillion ETL Auth Config. - * - * Choose between mysql and postgres connection for alation database - */ -export interface ConfigConnection { - /** - * Database of the data source. - * - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Host and port of the Hana service. - * - * Host and port of the source service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Matillion Host - */ - hostPort?: string; - /** - * Password to connect to Hana. - * - * Password for Superset. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to the Matillion. - */ - password?: string; - /** - * Username to connect to Hana. This user should have privileges to read all the metadata. - * - * Username for Superset. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to the Matillion. This user should have privileges to read all the - * metadata in Matillion. - */ - username?: string; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; - /** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ - provider?: Provider; - /** - * SSL Configuration details. - */ - sslConfig?: ConnectionSSLConfig; - verifySSL?: VerifySSL; - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: ConnectionScheme; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: ConnectionType; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - supportsViewLineageExtraction?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ -export enum Provider { - DB = "db", - LDAP = "ldap", -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum ConnectionScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", - SqlitePysqlite = "sqlite+pysqlite", -} - -/** - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - */ -export interface ConnectionSSLConfig { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum ConnectionType { - Backend = "Backend", - MatillionETL = "MatillionETL", - Mysql = "Mysql", - Postgres = "Postgres", - SQLite = "SQLite", -} - -/** - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * initial Kafka consumer offset - */ -export enum InitialConsumerOffsets { - Earliest = "earliest", - Latest = "latest", -} - -/** - * GCP Credentials - * - * GCP credentials configs. - * - * GCP Credentials for Google Drive API - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Underlying database connection - * - * Mssql Database Connection Config - */ -export interface DatabaseConnectionClass { - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Host and port of the MSSQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to MSSQL. - */ - password?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: MssqlScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: MssqlType; - /** - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - */ - username?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum MssqlScheme { - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", -} - -/** - * Service Type - * - * Service type. - */ -export enum MssqlType { - Mssql = "Mssql", -} - -/** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - * - * Snowplow deployment type - */ -export enum SnowplowDeployment { - Bdp = "BDP", - Community = "Community", -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - * - * Do not set any credentials. Note that credentials are required to extract .lkml views and - * their lineage. - * - * Credentials for a GitHub repository - * - * Credentials for a BitBucket repository - * - * Credentials for a Gitlab repository - */ -export interface GitHubCredentials { - repositoryName?: string; - repositoryOwner?: string; - token?: string; - /** - * Credentials Type - */ - type?: GitHubCredentialsType; - /** - * Main production branch of the repository. E.g., `main` - */ - branch?: string; -} - -/** - * Credentials Type - * - * GitHub Credentials type - * - * BitBucket Credentials type - * - * Gitlab Credentials type - */ -export enum GitHubCredentialsType { - BitBucket = "BitBucket", - GitHub = "GitHub", - Gitlab = "Gitlab", -} - -/** - * The authentication method that the user uses to sign in. - */ -export enum IdentityType { - Anonymous = "ANONYMOUS", - Iam = "IAM", - Quicksight = "QUICKSIGHT", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: ConsumerConfigSSLClass; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * We support username/password or client certificate authentication - * - * Configuration for connecting to Nifi Basic Auth. - * - * Configuration for connecting to Nifi Client Certificate Auth. - */ -export interface NifiCredentialsConfiguration { - /** - * Nifi password to authenticate to the API. - */ - password?: string; - /** - * Nifi user to authenticate to the API. - */ - username?: string; - /** - * Boolean marking if we need to verify the SSL certs for Nifi. False by default. - */ - verifySSL?: boolean; - /** - * Path to the root CA certificate - */ - certificateAuthorityPath?: string; - /** - * Path to the client certificate - */ - clientCertificatePath?: string; - /** - * Path to the client key - */ - clientkeyPath?: string; -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * S3 Connection. - */ -export interface S3Connection { - awsConfig: AWSCredentials; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: S3Type; -} - -/** - * Service Type - * - * S3 service type - */ -export enum S3Type { - S3 = "S3", -} - -/** - * Source to get the .pbit files to extract lineage information - * - * Local config source where no extra information needs to be sent. - * - * Azure storage config for pbit files - * - * GCS storage config for pbit files - * - * S3 storage config for pbit files - */ -export interface PowerBIPbitFilesSource { - /** - * Directory path for the pbit files - */ - path?: string; - /** - * pbit File Configuration type - */ - pbitFileConfigType?: PbitFileConfigType; - /** - * Path of the folder where the .pbit files will be unzipped and datamodel schema will be - * extracted - */ - pbitFilesExtractDir?: string; - prefixConfig?: BucketDetails; - securityConfig?: SecurityConfigClass; -} - -/** - * pbit File Configuration type - */ -export enum PbitFileConfigType { - Azure = "azure", - Gcs = "gcs", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the .pbit files are stored - */ -export interface BucketDetails { - /** - * Name of the bucket where the .pbit files are stored - */ - bucketName?: string; - /** - * Path of the folder where the .pbit files are stored - */ - objectPrefix?: string; -} - -/** - * This schema publisher run modes. - */ -export enum RunMode { - Batch = "batch", - Stream = "stream", -} - -/** - * SASL Configuration details. - * - * SASL client configuration. - */ -export interface SASLClientConfig { - /** - * SASL security mechanism - */ - saslMechanism?: SaslMechanismType; - /** - * The SASL authentication password. - */ - saslPassword?: string; - /** - * The SASL authentication username. - */ - saslUsername?: string; -} - -/** - * sasl.mechanism Consumer Config property - * - * SASL Mechanism consumer config property - * - * SASL security mechanism - */ -export enum SaslMechanismType { - Gssapi = "GSSAPI", - Oauthbearer = "OAUTHBEARER", - Plain = "PLAIN", - ScramSHA256 = "SCRAM-SHA-256", - ScramSHA512 = "SCRAM-SHA-512", -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - * - * Secrets Manager Provider to use for fetching secrets. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * security.protocol consumer config property - * - * Kafka security protocol config - */ -export enum KafkaSecurityProtocol { - Plaintext = "PLAINTEXT", - SSL = "SSL", - SaslPlaintext = "SASL_PLAINTEXT", - SaslSSL = "SASL_SSL", -} - -export enum SpaceType { - Data = "Data", - Managed = "Managed", - Personal = "Personal", - Shared = "Shared", -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Config - */ -export interface SSLConfigObject { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; - /** - * SSL Certificates - */ - certificates?: SSLCertificates; - [property: string]: any; -} - -/** - * SSL Certificates - * - * SSL Configuration details. - * - * SSL Certificates By Path - * - * SSL Certificates By Values - */ -export interface SSLCertificates { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * CA Certificate Value - */ - caCertValue?: string; - /** - * Client Certificate Value - */ - clientCertValue?: string; - /** - * Private Key Value - */ - privateKeyValue?: string; - /** - * Staging Directory Path - */ - stagingDir?: string; -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * REST API Type - * - * REST API type - * - * Service Type - * - * Service type. - * - * service type - * - * Custom database service type - * - * Looker service type - * - * Metabase service type - * - * PowerBI service type - * - * PowerBIReportServer service type - * - * Redash service type - * - * Superset service type - * - * Tableau service type - * - * Mode service type - * - * Custom dashboard service type - * - * QuickSight service type - * - * Qlik sense service type - * - * Lightdash service type - * - * MicroStrategy service type - * - * Qlik Cloud service type - * - * Sigma service type - * - * ThoughtSpot service type - * - * Grafana service type - * - * Kafka service type - * - * Redpanda service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Amundsen service type - * - * Metadata to Elastic Search type - * - * OpenMetadata service type - * - * S3 service type - * - * ADLS service type - * - * Gcs service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * ElasticSearch service type - * - * OpenSearch Type - * - * OpenSearch service type - * - * Custom search service type - * - * Apache Ranger service type - * - * Google Drive service type - * - * SharePoint service type - * - * Custom Drive service type - */ -export enum ConfigType { - Adls = "ADLS", - Airbyte = "Airbyte", - Airflow = "Airflow", - Alation = "Alation", - AlationSink = "AlationSink", - Amundsen = "Amundsen", - Athena = "Athena", - Atlas = "Atlas", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDashboard = "CustomDashboard", - CustomDatabase = "CustomDatabase", - CustomDrive = "CustomDrive", - CustomMessaging = "CustomMessaging", - CustomMlModel = "CustomMlModel", - CustomPipeline = "CustomPipeline", - CustomSearch = "CustomSearch", - CustomStorage = "CustomStorage", - DBTCloud = "DBTCloud", - Dagster = "Dagster", - DataFactory = "DataFactory", - Databricks = "Databricks", - DatabricksPipeline = "DatabricksPipeline", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDashboard = "DomoDashboard", - DomoDatabase = "DomoDatabase", - DomoPipeline = "DomoPipeline", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - ElasticSearch = "ElasticSearch", - Epic = "Epic", - Exasol = "Exasol", - Fivetran = "Fivetran", - Flink = "Flink", - Gcs = "GCS", - Glue = "Glue", - GluePipeline = "GluePipeline", - GoogleDrive = "GoogleDrive", - Grafana = "Grafana", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - Kafka = "Kafka", - KafkaConnect = "KafkaConnect", - Kinesis = "Kinesis", - Lightdash = "Lightdash", - Looker = "Looker", - MariaDB = "MariaDB", - Matillion = "Matillion", - Metabase = "Metabase", - MetadataES = "MetadataES", - MicroStrategy = "MicroStrategy", - Mlflow = "Mlflow", - Mode = "Mode", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Nifi = "Nifi", - OpenLineage = "OpenLineage", - OpenMetadata = "OpenMetadata", - OpenSearch = "OpenSearch", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - PowerBI = "PowerBI", - PowerBIReportServer = "PowerBIReportServer", - Presto = "Presto", - QlikCloud = "QlikCloud", - QlikSense = "QlikSense", - QuickSight = "QuickSight", - REST = "Rest", - Ranger = "Ranger", - Redash = "Redash", - Redpanda = "Redpanda", - Redshift = "Redshift", - S3 = "S3", - SAS = "SAS", - SQLite = "SQLite", - SageMaker = "SageMaker", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SharePoint = "SharePoint", - Sigma = "Sigma", - SingleStore = "SingleStore", - Sklearn = "Sklearn", - Snowflake = "Snowflake", - Snowplow = "Snowplow", - Spark = "Spark", - Spline = "Spline", - Ssas = "SSAS", - Ssis = "SSIS", - Stitch = "Stitch", - Superset = "Superset", - Synapse = "Synapse", - Tableau = "Tableau", - Teradata = "Teradata", - ThoughtSpot = "ThoughtSpot", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - VertexAI = "VertexAI", - Vertica = "Vertica", - Wherescape = "Wherescape", -} - -/** - * Operation to be performed on the entity - */ -export interface Operation { - /** - * Entity to be modified - */ - entityLink: string; - /** - * The id of the operation - */ - id: string; - /** - * The configuration for the operation to be applied - */ - parameters: ReverseIngestionConfig; - /** - * Templated SQL command to be used for the operation. Context parameters will be populated - * based on the event type. - */ - SQLTemplate?: string; - /** - * Type of operation to perform - */ - type: Type; -} - -/** - * The configuration for the operation to be applied - * - * Configuration for updating descriptions - * - * Configuration for updating owners - * - * Configuration for updating tags - */ -export interface ReverseIngestionConfig { - /** - * New description of the service - */ - newDescription?: string; - /** - * Previous description of the service - */ - previousDescription?: string; - /** - * Added owners to be applied - */ - addedOwners?: EntityReference[]; - /** - * Removed owners from the entity - */ - removedOwners?: EntityReference[]; - /** - * Added tags to be applied - */ - addedTags?: TagLabel[]; - /** - * Removed tags of the entity - */ - removedTags?: TagLabel[]; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - */ -export interface TagLabel { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelType; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelType { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * Type of operation to perform - */ -export enum Type { - UpdateDescription = "UPDATE_DESCRIPTION", - UpdateOwner = "UPDATE_OWNER", - UpdateTags = "UPDATE_TAGS", -} - -/** - * Type of service such as Database, Dashboard, Messaging, etc. - * - * This schema defines the service types entities which requires a connection. - */ -export enum ServiceType { - API = "Api", - Dashboard = "Dashboard", - Database = "Database", - Drive = "Drive", - Messaging = "Messaging", - Metadata = "Metadata", - MlModel = "MlModel", - Pipeline = "Pipeline", - Search = "Search", - Security = "Security", - Storage = "Storage", -} - -/** - * Pipeline type - * - * Reverse Ingestion Config Pipeline type - */ -export enum ReverseIngestionType { - ReverseIngestion = "ReverseIngestion", -} - -/** - * Response to the request. - * - * TestConnectionResult is the definition that will encapsulate result of running the test - * connection steps. - * - * Apply a set of operations on a service - */ -export interface TestConnectionResult { - /** - * Last time that the test connection was executed - */ - lastUpdatedAt?: number; - /** - * Test Connection Result computation status. - */ - status?: StatusType; - /** - * Steps to test the connection. Order matters. - */ - steps?: TestConnectionStepResult[]; - /** - * Error message in case of failure - */ - message?: string; - /** - * List of operations to be performed on the service - */ - results?: ReverseIngestionOperationResult[]; - /** - * The id of the service to be modified - */ - serviceId?: string; - /** - * Whether the workflow was successful. Failure indicates a critical failure such as - * connection issues. - */ - success?: boolean; -} - -export interface ReverseIngestionOperationResult { - /** - * The id of the operation - */ - id: string; - /** - * Error message in case of failure - */ - message?: string; - /** - * Whether the specific operation was successful - */ - success: boolean; - [property: string]: any; -} - -/** - * Test Connection Result computation status. - * - * Enum defining possible Test Connection Result status - */ -export enum StatusType { - Failed = "Failed", - Running = "Running", - Successful = "Successful", -} - -/** - * Function that tests one specific element of the service. E.g., listing schemas, lineage, - * or tags. - */ -export interface TestConnectionStepResult { - /** - * In case of failed step, this field would contain the actual error faced during the step. - */ - errorLog?: string; - /** - * Is this step mandatory to be passed? - */ - mandatory: boolean; - /** - * Results or exceptions to be shared after running the test. This message comes from the - * test connection definition - */ - message?: string; - /** - * Name of the step being tested - */ - name: string; - /** - * Did the step pass successfully? - */ - passed: boolean; -} - -/** - * Workflow computation status. - * - * Enum defining possible Workflow status - */ -export enum WorkflowStatus { - Failed = "Failed", - Pending = "Pending", - Running = "Running", - Successful = "Successful", -} - -/** - * Type of the workflow. - * - * This enum defines the type for which this workflow applies to. - */ -export enum WorkflowType { - QueryRunner = "QUERY_RUNNER", - ReverseIngestion = "REVERSE_INGESTION", - TestConnection = "TEST_CONNECTION", - TestSparkEngineConnection = "TEST_SPARK_ENGINE_CONNECTION", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts deleted file mode 100644 index 8310f33d7df..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts +++ /dev/null @@ -1,2285 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Create Database service entity request - */ -export interface CreateDatabaseService { - connection?: DatabaseConnection; - /** - * List of fully qualified names of data products this entity is part of. - */ - dataProducts?: string[]; - /** - * Description of Database entity. - */ - description?: string; - /** - * Display Name that identifies this database service. - */ - displayName?: string; - /** - * Fully qualified names of the domains the Database Service belongs to. - */ - domains?: string[]; - /** - * The ingestion agent responsible for executing the ingestion pipeline. It will be defined - * at runtime based on the Ingestion Agent of the service. - */ - ingestionRunner?: EntityReference; - /** - * Name that identifies the this entity instance uniquely - */ - name: string; - /** - * Owners of this database service. - */ - owners?: EntityReference[]; - serviceType: DatabaseServiceType; - /** - * Tags for this Database Service. - */ - tags?: TagLabel[]; -} - -/** - * Database Connection. - */ -export interface DatabaseConnection { - config?: ConfigObject; -} - -/** - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - */ -export interface ConfigObject { - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - */ - hostPort?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Service Type - * - * Custom database service type - */ - type?: ConfigType; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - awsConfig?: AWSCredentials; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to the Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - */ - password?: string; - /** - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to the Salesforce. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - */ - username?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - */ - authType?: AuthenticationType | NoConfigAuthenticationTypes; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - */ - connectionTimeout?: number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - */ - configSource?: TaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - */ - auth?: AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * SSL Configuration details. - */ - sslConfig?: Config; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Access token to connect to DOMO - */ - accessToken?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * Client ID for DOMO - * - * Azure Application (client) ID for service principal authentication. - */ - clientId?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - */ - secretToken?: string; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * Choose between Database connection or HDB User Store connection. - */ - connection?: SAPHanaConnection; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Generated Token to connect to Databricks. - */ - token?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * API key to authenticate with the SAP ERP APIs. - */ - apiKey?: string; - /** - * Pagination limit used while querying the SAP ERP API for fetching the entities - */ - paginationLimit?: number; - verifySSL?: VerifySSL; - /** - * Azure Application client secret for service principal authentication. - */ - clientSecret?: string; - /** - * Azure Directory (tenant) ID for service principal authentication. - */ - tenantId?: string; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - [property: string]: any; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface AuthenticationType { - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: Authentication; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum Authentication { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: Credentials | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface TaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConnectionClass; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: SecurityConfigClass; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConnectionClass { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Azure Cloud Credentials - * - * GCP Credentials - * - * GCP credentials configs. - * - * AWS credentials configs. - */ -export interface SecurityConfigClass { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - */ -export interface SAPHanaConnection { - /** - * Database of the data source. - */ - database?: string; - /** - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Host and port of the Hana service. - */ - hostPort?: string; - /** - * Password to connect to Hana. - */ - password?: string; - /** - * Username to connect to Hana. This user should have privileges to read all the metadata. - */ - username?: string; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; -} - -/** - * GCP Credentials - * - * GCP credentials configs. - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; -} - -/** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: Config; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface Config { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * Service Type - * - * Service type. - * - * service type - * - * Custom database service type - */ -export enum ConfigType { - Athena = "Athena", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDatabase = "CustomDatabase", - Databricks = "Databricks", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDatabase = "DomoDatabase", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - Epic = "Epic", - Exasol = "Exasol", - Glue = "Glue", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - MariaDB = "MariaDB", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - Presto = "Presto", - Redshift = "Redshift", - SAS = "SAS", - SQLite = "SQLite", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SingleStore = "SingleStore", - Snowflake = "Snowflake", - Ssas = "SSAS", - Synapse = "Synapse", - Teradata = "Teradata", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - Vertica = "Vertica", -} - -/** - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * The ingestion agent responsible for executing the ingestion pipeline. It will be defined - * at runtime based on the Ingestion Agent of the service. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Owners of this database service. - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres... - */ -export enum DatabaseServiceType { - Athena = "Athena", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDatabase = "CustomDatabase", - Databricks = "Databricks", - Datalake = "Datalake", - Db2 = "Db2", - Dbt = "Dbt", - DeltaLake = "DeltaLake", - DomoDatabase = "DomoDatabase", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - Epic = "Epic", - Exasol = "Exasol", - Glue = "Glue", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - MariaDB = "MariaDB", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - Presto = "Presto", - QueryLog = "QueryLog", - Redshift = "Redshift", - SAS = "SAS", - SQLite = "SQLite", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SingleStore = "SingleStore", - Snowflake = "Snowflake", - Ssas = "SSAS", - Synapse = "Synapse", - Teradata = "Teradata", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - Vertica = "Vertica", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - */ -export interface TagLabel { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelType; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelType { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts deleted file mode 100644 index 98a0b96ed57..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts +++ /dev/null @@ -1,6154 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Ingestion Pipeline Config is used to set up an Airflow DAG. - */ -export interface CreateIngestionPipeline { - airflowConfig: AirflowConfig; - /** - * Description of the pipeline. - */ - description?: string; - /** - * Display Name that identifies this ingestion pipeline. - */ - displayName?: string; - /** - * Fully qualified names of the domains the Ingestion Pipeline belongs to. - */ - domains?: string[]; - /** - * Set the logging level for the workflow. - */ - loggerLevel?: LogLevels; - /** - * Name that identifies this pipeline instance uniquely. - */ - name: string; - /** - * Owner of this Ingestion Pipeline. - */ - owners?: EntityReference[]; - pipelineType: PipelineType; - /** - * The processing engine responsible for executing the ingestion pipeline logic. - */ - processingEngine?: EntityReference; - provider?: ProviderType; - /** - * Control if we want to flag the workflow as failed if we encounter any processing errors. - */ - raiseOnError?: boolean; - /** - * Link to the service for which ingestion pipeline is ingesting the metadata. - */ - service: EntityReference; - sourceConfig: SourceConfig; -} - -/** - * Properties to configure the Airflow pipeline that will run the workflow. - */ -export interface AirflowConfig { - /** - * Concurrency of the Pipeline. - */ - concurrency?: number; - /** - * Email to notify workflow status. - */ - email?: string; - /** - * End Date of the pipeline. - */ - endDate?: Date; - /** - * Maximum Number of active runs. - */ - maxActiveRuns?: number; - /** - * pause the pipeline from running once the deploy is finished successfully. - */ - pausePipeline?: boolean; - /** - * Run past executions if the start date is in the past. - */ - pipelineCatchup?: boolean; - /** - * Timezone in which pipeline going to be scheduled. - */ - pipelineTimezone?: string; - /** - * Retry pipeline in case of failure. - */ - retries?: number; - /** - * Delay between retries in seconds. - */ - retryDelay?: number; - /** - * Scheduler Interval for the pipeline in cron format. - */ - scheduleInterval?: string; - /** - * Start date of the pipeline. - */ - startDate?: Date; - /** - * Default view in Airflow. - */ - workflowDefaultView?: string; - /** - * Default view Orientation in Airflow. - */ - workflowDefaultViewOrientation?: string; - /** - * Timeout for the workflow in seconds. - */ - workflowTimeout?: number; -} - -/** - * Set the logging level for the workflow. - * - * Supported logging levels - */ -export enum LogLevels { - Debug = "DEBUG", - Error = "ERROR", - Info = "INFO", - Warn = "WARN", -} - -/** - * Owner of this Ingestion Pipeline. - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * The processing engine responsible for executing the ingestion pipeline logic. - * - * Link to the service for which ingestion pipeline is ingesting the metadata. - * - * Service to be modified - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Type of Pipeline - metadata, usage - */ -export enum PipelineType { - Application = "application", - AutoClassification = "autoClassification", - DataInsight = "dataInsight", - Dbt = "dbt", - ElasticSearchReindex = "elasticSearchReindex", - Lineage = "lineage", - Metadata = "metadata", - Profiler = "profiler", - TestSuite = "TestSuite", - Usage = "usage", -} - -/** - * Type of provider of an entity. Some entities are provided by the `system`. Some are - * entities created and provided by the `user`. Typically `system` provide entities can't be - * deleted and can only be disabled. Some apps such as AutoPilot create entities with - * `automation` provider type. These entities can be deleted by the user. - */ -export enum ProviderType { - Automation = "automation", - System = "system", - User = "user", -} - -/** - * Additional connection configuration. - */ -export interface SourceConfig { - config?: Pipeline; -} - -/** - * DatabaseService Metadata Pipeline Configuration. - * - * DatabaseService Query Usage Pipeline Configuration. - * - * DatabaseService Query Lineage Pipeline Configuration. - * - * DashboardService Metadata Pipeline Configuration. - * - * MessagingService Metadata Pipeline Configuration. - * - * DatabaseService Profiler Pipeline Configuration. - * - * DatabaseService AutoClassification & Auto Classification Pipeline Configuration. - * - * PipelineService Metadata Pipeline Configuration. - * - * MlModelService Metadata Pipeline Configuration. - * - * StorageService Metadata Pipeline Configuration. - * - * DriveService Metadata Pipeline Configuration. - * - * SearchService Metadata Pipeline Configuration. - * - * TestSuite Pipeline Configuration. - * - * Data Insight Pipeline Configuration. - * - * DBT Pipeline Configuration. - * - * Application Pipeline Configuration. - * - * ApiService Metadata Pipeline Configuration. - * - * Apply a set of operations on a service - */ -export interface Pipeline { - /** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Optional configuration to toggle the DDL Statements ingestion. - */ - includeDDL?: boolean; - /** - * Set the 'Include Owners' toggle to control whether to include owners to the ingested - * entity if the owner email matches with a user stored in the OM server as part of metadata - * ingestion. If the ingested entity already exists and has an owner, the owner will not be - * overwritten. - * - * Enabling a flag will replace the current owner with a new owner from the source during - * metadata ingestion, if the current owner is null. It is recommended to keep the flag - * enabled to obtain the owner information during the first metadata ingestion. - */ - includeOwners?: boolean; - /** - * Optional configuration to toggle the Stored Procedures ingestion. - */ - includeStoredProcedures?: boolean; - /** - * Optional configuration to turn off fetching metadata for tables. - */ - includeTables?: boolean; - /** - * Optional configuration to toggle the tags ingestion. - */ - includeTags?: boolean; - /** - * Optional configuration to turn off fetching metadata for views. - */ - includeViews?: boolean; - /** - * Use incremental Metadata extraction after the first execution. This is commonly done by - * getting the changes from Audit tables on the supporting databases. - */ - incremental?: IncrementalMetadataExtractionConfiguration; - /** - * Optional configuration to soft delete databases in OpenMetadata if the source databases - * are deleted. Also, if the database is deleted, all the associated entities like schemas, - * tables, views, stored procedures, lineage, etc., with that database will be deleted - */ - markDeletedDatabases?: boolean; - /** - * Optional configuration to soft delete schemas in OpenMetadata if the source schemas are - * deleted. Also, if the schema is deleted, all the associated entities like tables, views, - * stored procedures, lineage, etc., with that schema will be deleted - */ - markDeletedSchemas?: boolean; - /** - * Optional configuration to soft delete stored procedures in OpenMetadata if the source - * stored procedures are deleted. Also, if the stored procedures is deleted, all the - * associated entities like lineage, etc., with that stored procedures will be deleted - */ - markDeletedStoredProcedures?: boolean; - /** - * This is an optional configuration for enabling soft deletion of tables. When this option - * is enabled, only tables that have been deleted from the source will be soft deleted, and - * this will apply solely to the schema that is currently being ingested via the pipeline. - * Any related entities such as test suites or lineage information that were associated with - * those tables will also be deleted. - */ - markDeletedTables?: boolean; - /** - * Set the 'Override Metadata' toggle to control whether to override the existing metadata - * in the OpenMetadata server with the metadata fetched from the source. If the toggle is - * set to true, the metadata fetched from the source will override the existing metadata in - * the OpenMetadata server. If the toggle is set to false, the metadata fetched from the - * source will not override the existing metadata in the OpenMetadata server. This is - * applicable for fields like description, tags, owner and displayName - */ - overrideMetadata?: boolean; - /** - * Configuration to tune how far we want to look back in query logs to process Stored - * Procedures results. - * - * Configuration to tune how far we want to look back in query logs to process usage data. - * - * Configuration to tune how far we want to look back in query logs to process lineage data. - */ - queryLogDuration?: number; - /** - * Configuration to set the timeout for parsing the query in seconds. - */ - queryParsingTimeoutLimit?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only fetch tables or databases that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex exclude tables or databases that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Number of Threads to use in order to parallelize Table ingestion. - * - * Number of Threads to use in order to parallelize lineage ingestion. - * - * Number of Threads to use in order to parallelize Drive ingestion. - */ - threads?: number; - /** - * Pipeline type - */ - type?: FluffyType; - /** - * Regex will be applied on fully qualified name (e.g - * service_name.db_name.schema_name.table_name) instead of raw name (e.g. table_name) - * - * Regex will be applied on fully qualified name (e.g service_name.directory_name.file_name) - * instead of raw name (e.g. file_name) - */ - useFqnForFiltering?: boolean; - /** - * Configuration the condition to filter the query history. - */ - filterCondition?: string; - /** - * Configuration to process query cost - */ - processQueryCostAnalysis?: boolean; - /** - * Configuration to set the file path for query logs - */ - queryLogFilePath?: string; - /** - * Configuration to set the limit for query logs - */ - resultLimit?: number; - /** - * Temporary file name to store the query logs before processing. Absolute file path - * required. - */ - stageFileLocation?: string; - /** - * Set 'Cross Database Service Names' to process lineage with the database. - */ - crossDatabaseServiceNames?: string[]; - /** - * Handle Lineage for Snowflake Temporary and Transient Tables. - */ - enableTempTableLineage?: boolean; - /** - * Set the 'Incremental Lineage Processing' toggle to control whether to process lineage - * incrementally. - */ - incrementalLineageProcessing?: boolean; - /** - * Set the 'Override View Lineage' toggle to control whether to override the existing view - * lineage. - */ - overrideViewLineage?: boolean; - /** - * Configuration to set the timeout for parsing the query in seconds. - */ - parsingTimeoutLimit?: number; - /** - * Set the 'Process Cross Database Lineage' toggle to control whether to process table - * lineage across different databases. - */ - processCrossDatabaseLineage?: boolean; - /** - * Set the 'Process Query Lineage' toggle to control whether to process query lineage. - */ - processQueryLineage?: boolean; - /** - * Set the 'Process Stored ProcedureLog Lineage' toggle to control whether to process stored - * procedure lineage. - */ - processStoredProcedureLineage?: boolean; - /** - * Set the 'Process View Lineage' toggle to control whether to process view lineage. - */ - processViewLineage?: boolean; - /** - * Regex to only fetch stored procedures that matches the pattern. - */ - storedProcedureFilterPattern?: FilterPattern; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Optional configuration to toggle the ingestion of data models. - */ - includeDataModels?: boolean; - /** - * Optional Configuration to include/exclude draft dashboards. By default it will include - * draft dashboards - */ - includeDraftDashboard?: boolean; - /** - * Details required to generate Lineage - */ - lineageInformation?: LineageInformation; - /** - * Optional configuration to soft delete dashboards in OpenMetadata if the source dashboards - * are deleted. Also, if the dashboard is deleted, all the associated entities like lineage, - * etc., with that dashboard will be deleted - */ - markDeletedDashboards?: boolean; - /** - * Optional configuration to soft delete data models in OpenMetadata if the source data - * models are deleted. Also, if the data models is deleted, all the associated entities like - * lineage, etc., with that data models will be deleted - */ - markDeletedDataModels?: boolean; - /** - * Set the 'Override Lineage' toggle to control whether to override the existing lineage. - */ - overrideLineage?: boolean; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * Option to turn on/off generating sample data during metadata extraction. - */ - generateSampleData?: boolean; - /** - * Optional configuration to soft delete topics in OpenMetadata if the source topics are - * deleted. Also, if the topic is deleted, all the associated entities like sample data, - * lineage, etc., with that topic will be deleted - */ - markDeletedTopics?: boolean; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Regex to only compute metrics for table that matches the given tag, tiers, gloassary - * pattern. - */ - classificationFilterPattern?: FilterPattern; - /** - * Option to turn on/off column metric computation. If enabled, profiler will compute column - * level metrics. - */ - computeColumnMetrics?: boolean; - /** - * Option to turn on/off computing profiler metrics. - */ - computeMetrics?: boolean; - /** - * Option to turn on/off table metric computation. If enabled, profiler will compute table - * level metrics. - */ - computeTableMetrics?: boolean; - processingEngine?: ProcessingEngine; - /** - * Percentage of data or no. of rows used to compute the profiler metrics and run data - * quality tests - * - * Percentage of data or no. of rows we want to execute the profiler and tests on - */ - profileSample?: number; - profileSampleType?: ProfileSampleType; - /** - * Whether to randomize the sample data or not. - */ - randomizedSample?: boolean; - samplingMethodType?: SamplingMethodType; - /** - * Number of threads to use during metric computations - */ - threadCount?: number; - /** - * Profiler Timeout in Seconds - */ - timeoutSeconds?: number; - /** - * Use system tables to extract metrics. Metrics that cannot be gathered from system tables - * will use the default methods. Using system tables can be faster but requires gathering - * statistics before running (for example using the ANALYZE procedure). More information can - * be found in the documentation: https://docs.openmetadata.org/latest/profler - */ - useStatistics?: boolean; - /** - * Set the Confidence value for which you want the column to be tagged as PII. Confidence - * value ranges from 0 to 100. A higher number will yield less false positives but more - * false negatives. A lower number will yield more false positives but less false negatives. - */ - confidence?: number; - /** - * Optional configuration to automatically tag columns that might contain sensitive - * information - */ - enableAutoClassification?: boolean; - /** - * Number of sample rows to ingest when 'Generate Sample Data' is enabled - */ - sampleDataCount?: number; - /** - * Option to turn on/off storing sample data. If enabled, we will ingest sample data for - * each table. - */ - storeSampleData?: boolean; - /** - * Optional configuration to turn off fetching lineage from pipelines. - */ - includeLineage?: boolean; - /** - * Optional configuration to toggle whether the un-deployed pipelines should be ingested or - * not. If set to false, only deployed pipelines will be ingested. - */ - includeUnDeployedPipelines?: boolean; - /** - * Optional configuration to soft delete Pipelines in OpenMetadata if the source Pipelines - * are deleted. Also, if the Pipeline is deleted, all the associated entities like lineage, - * etc., with that Pipeline will be deleted - */ - markDeletedPipelines?: boolean; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete MlModels in OpenMetadata if the source MlModels are - * deleted. Also, if the MlModel is deleted, all the associated entities like lineage, etc., - * with that MlModels will be deleted - */ - markDeletedMlModels?: boolean; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete containers in OpenMetadata if the source containers - * are deleted. Also, if the topic is deleted, all the associated entities with that - * containers will be deleted - */ - markDeletedContainers?: boolean; - storageMetadataConfigSource?: StorageMetadataConfigurationSource; - /** - * Regex to only include/exclude directories that matches the pattern. - */ - directoryFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude files that matches the pattern. - */ - fileFilterPattern?: FilterPattern; - /** - * Optional configuration to turn off fetching metadata for directories. - */ - includeDirectories?: boolean; - /** - * Optional configuration to turn off fetching metadata for files. - */ - includeFiles?: boolean; - /** - * Optional configuration to turn off fetching metadata for spreadsheets. - */ - includeSpreadsheets?: boolean; - /** - * Optional configuration to turn off fetching metadata for worksheets. - */ - includeWorksheets?: boolean; - /** - * Optional configuration to soft delete directories in OpenMetadata if the source - * directories are deleted. Also, if the directory is deleted, all the associated entities - * like files, spreadsheets, worksheets, lineage, etc., with that directory will be deleted - */ - markDeletedDirectories?: boolean; - /** - * Optional configuration to soft delete files in OpenMetadata if the source files are - * deleted. Also, if the file is deleted, all the associated entities like lineage, etc., - * with that file will be deleted - */ - markDeletedFiles?: boolean; - /** - * Optional configuration to soft delete spreadsheets in OpenMetadata if the source - * spreadsheets are deleted. Also, if the spreadsheet is deleted, all the associated - * entities like worksheets, lineage, etc., with that spreadsheet will be deleted - */ - markDeletedSpreadsheets?: boolean; - /** - * Optional configuration to soft delete worksheets in OpenMetadata if the source worksheets - * are deleted. Also, if the worksheet is deleted, all the associated entities like lineage, - * etc., with that worksheet will be deleted - */ - markDeletedWorksheets?: boolean; - /** - * Regex to only include/exclude spreadsheets that matches the pattern. - */ - spreadsheetFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude worksheets that matches the pattern. - */ - worksheetFilterPattern?: FilterPattern; - /** - * Enable the 'Include Index Template' toggle to manage the ingestion of index template data. - */ - includeIndexTemplate?: boolean; - /** - * Optional configuration to turn off fetching sample data for search index. - */ - includeSampleData?: boolean; - /** - * Optional configuration to soft delete search indexes in OpenMetadata if the source search - * indexes are deleted. Also, if the search index is deleted, all the associated entities - * like lineage, etc., with that search index will be deleted - */ - markDeletedSearchIndexes?: boolean; - /** - * No. of records of sample data we want to ingest. - */ - sampleSize?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Fully qualified name of the entity to be tested, if we're working with a basic suite. - */ - entityFullyQualifiedName?: string; - /** - * Service connections to be used for the logical test suite. - */ - serviceConnections?: ServiceConnections[]; - /** - * List of test cases to be executed on the entity. If null, all test cases will be executed. - */ - testCases?: string[]; - /** - * Maximum number of events entities in a batch (Default 1000). - */ - batchSize?: number; - /** - * Certificate path to be added in configuration. The path should be local in the Ingestion - * Container. - */ - caCerts?: string; - recreateIndex?: boolean; - /** - * Region name. Required when using AWS Credentials. - */ - regionName?: string; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Connection Timeout - */ - timeout?: number; - /** - * Indicates whether to use aws credentials when connecting to OpenSearch in AWS. - */ - useAwsCredentials?: boolean; - /** - * Indicates whether to use SSL when connecting to ElasticSearch. By default, we will ignore - * SSL settings. - */ - useSSL?: boolean; - /** - * Indicates whether to verify certificates when using SSL connection to ElasticSearch. - * Ignored by default. Is set to true, make sure to send the certificates in the property - * `CA Certificates`. - */ - verifyCerts?: boolean; - /** - * Custom OpenMetadata Classification name for dbt tags. - */ - dbtClassificationName?: string; - /** - * Available sources to fetch DBT catalog and manifest files. - */ - dbtConfigSource?: DBTConfigurationSource; - /** - * Optional configuration to update the description from DBT or not - */ - dbtUpdateDescriptions?: boolean; - /** - * Optional configuration to update the owners from DBT or not - */ - dbtUpdateOwners?: boolean; - /** - * Optional configuration to search across databases for tables or not - */ - searchAcrossDatabases?: boolean; - /** - * Regex to only fetch tags that matches the pattern. - */ - tagFilterPattern?: FilterPattern; - /** - * Application configuration - */ - appConfig?: any[] | boolean | number | null | CollateAIAppConfig | string; - /** - * Application private configuration - */ - appPrivateConfig?: PrivateConfig; - /** - * Source Python Class Name to run the application - */ - sourcePythonClass?: string; - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete api collections in OpenMetadata if the source - * collections are deleted. Also, if the collection is deleted, all the associated entities - * like endpoints, etc., with that collection will be deleted - */ - markDeletedApiCollections?: boolean; - /** - * Optional value of the ingestion runner name responsible for running the workflow - */ - ingestionRunner?: string; - /** - * List of operations to be performed on the service - */ - operations?: Operation[]; - /** - * Service to be modified - */ - service?: EntityReference; -} - -/** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to only fetch databases that matches the pattern. - * - * Regex to only fetch tables or databases that matches the pattern. - * - * Regex to only fetch stored procedures that matches the pattern. - * - * Regex exclude tables or databases that matches the pattern. - * - * Regex exclude or include charts that matches the pattern. - * - * Regex to exclude or include dashboards that matches the pattern. - * - * Regex exclude or include data models that matches the pattern. - * - * Regex to exclude or include projects that matches the pattern. - * - * Regex to only fetch topics that matches the pattern. - * - * Regex to only compute metrics for table that matches the given tag, tiers, gloassary - * pattern. - * - * Regex exclude pipelines. - * - * Regex to only fetch MlModels with names matching the pattern. - * - * Regex to only fetch containers that matches the pattern. - * - * Regex to only include/exclude directories that matches the pattern. - * - * Regex to only include/exclude files that matches the pattern. - * - * Regex to only include/exclude spreadsheets that matches the pattern. - * - * Regex to only include/exclude worksheets that matches the pattern. - * - * Regex to only fetch search indexes that matches the pattern. - * - * Regex to only fetch api collections with names matching the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - * - * Regex to only fetch tags that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Configuration for the CollateAI External Application. - * - * Configuration for the Automator External Application. - * - * This schema defines the Slack App Token Configuration - * - * No configuration needed to instantiate the Data Insights Pipeline. The logic is handled - * in the backend. - * - * Search Indexing App. - * - * Configuration for the Collate AI Quality Agent. - * - * Configuration for the AutoPilot Application. - */ -export interface CollateAIAppConfig { - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - filter?: string; - /** - * Patch the description if it is empty, instead of raising a suggestion - */ - patchIfEmpty?: boolean; - /** - * Application Type - */ - type?: CollateAIAppConfigType; - /** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - */ - actions?: Action[]; - /** - * Entities selected to run the automation. - */ - resources?: Resource; - /** - * Bot Token - */ - botToken?: string; - /** - * User Token - */ - userToken?: string; - backfillConfiguration?: BackfillConfiguration; - /** - * Maximum number of events processed at a time (Default 100). - * - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - moduleConfiguration?: ModuleConfiguration; - /** - * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property - * Type and are facing errors. Bear in mind that recreating the index will delete your - * DataAssets and a backfill will be needed. - */ - recreateDataAssetsIndex?: boolean; - sendToAdmins?: boolean; - sendToTeams?: boolean; - /** - * Enable automatic performance tuning based on cluster capabilities and database entity - * count - */ - autoTune?: boolean; - /** - * Number of threads to use for reindexing - */ - consumerThreads?: number; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Initial backoff time in milliseconds - */ - initialBackoff?: number; - /** - * Maximum backoff time in milliseconds - */ - maxBackoff?: number; - /** - * Maximum number of concurrent requests to the search index - */ - maxConcurrentRequests?: number; - /** - * Maximum number of retries for a failed request - */ - maxRetries?: number; - /** - * Maximum number of events sent in a batch (Default 100). - */ - payLoadSize?: number; - /** - * Number of threads to use for reindexing - */ - producerThreads?: number; - /** - * Queue Size to user internally for reindexing. - */ - queueSize?: number; - /** - * This schema publisher run modes. - */ - recreateIndex?: boolean; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Whether the suggested tests should be active or not upon suggestion - * - * Whether the AutoPilot Workflow should be active or not. - */ - active?: boolean; - /** - * Enter the retention period for Activity Threads of type = 'Conversation' records in days - * (e.g., 30 for one month, 60 for two months). - */ - activityThreadsRetentionPeriod?: number; - /** - * Enter the retention period for change event records in days (e.g., 7 for one week, 30 for - * one month). - */ - changeEventRetentionPeriod?: number; - /** - * Service Entity Link for which to trigger the application. - */ - entityLink?: string; - [property: string]: any; -} - -/** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - * - * Apply Classification Tags to the selected assets. - * - * Remove Classification Tags Action Type - * - * Apply Glossary Terms to the selected assets. - * - * Remove Glossary Terms Action Type - * - * Add domains to the selected assets. - * - * Remove domains from the selected assets. - * - * Apply Tags to the selected assets. - * - * Add a Custom Property to the selected assets. - * - * Remove Owner Action Type - * - * Add an owner to the selected assets. - * - * Add Test Cases to the selected assets. - * - * Remove Test Cases Action Type - * - * Add owners to the selected assets. - * - * Remove Custom Properties Action Type - * - * Add a Data Product to the selected assets. - * - * Remove a Data Product to the selected assets. - * - * Propagate description, tags and glossary terms via lineage - * - * ML Tagging action configuration for external automator. - */ -export interface Action { - /** - * Apply tags to the children of the selected assets that match the criteria. E.g., columns, - * tasks, topic fields,... - * - * Remove tags from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply terms to the children of the selected assets that match the criteria. E.g., - * columns, tasks, topic fields,... - * - * Remove terms from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply the description to the children of the selected assets that match the criteria. - * E.g., columns, tasks, topic fields,... - * - * Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Add tests to the selected table columns - * - * Remove tests to the selected table columns - */ - applyToChildren?: string[]; - /** - * Update tags even if they are already defined in the asset. By default, incoming tags are - * merged with the existing ones. - * - * Update terms even if they are already defined in the asset. By default, incoming terms - * are merged with the existing ones. - * - * Update the domains even if they are defined in the asset. By default, we will only apply - * the domains to assets without domains. - * - * Update the description even if they are already defined in the asset. By default, we'll - * only add the descriptions to assets without the description set. - * - * Update the Custom Property even if it is defined in the asset. By default, we will only - * apply the owners to assets without the given Custom Property informed. - * - * Update the tier even if it is defined in the asset. By default, we will only apply the - * tier to assets without tier. - * - * Update the test even if it is defined in the asset. By default, we will only apply the - * test to assets without the existing test already existing. - * - * Update the owners even if it is defined in the asset. By default, we will only apply the - * owners to assets without owner. - * - * Update the Data Product even if the asset belongs to a different Domain. By default, we - * will only add the Data Product if the asset has no Domain, or it belongs to the same - * domain as the Data Product. - * - * Update descriptions, tags and Glossary Terms via lineage even if they are already defined - * in the asset. By default, descriptions are only updated if they are not already defined - * in the asset, and incoming tags are merged with the existing ones. - */ - overwriteMetadata?: boolean; - /** - * Classification Tags to apply (source must be 'Classification') - * - * Classification Tags to remove (source must be 'Classification') - */ - tags?: TierElement[]; - /** - * Application Type - */ - type: ActionType; - /** - * Remove tags from all the children and parent of the selected assets. - * - * Remove terms from all the children and parent of the selected assets. - * - * Remove descriptions from all the children and parent of the selected assets. - */ - applyToAll?: boolean; - /** - * Remove tags by its label type - * - * Remove terms by its label type - */ - labels?: LabelElement[]; - /** - * Glossary Terms to apply - * - * Glossary Terms to remove - */ - terms?: TierElement[]; - /** - * Domains to apply - */ - domains?: EntityReference[]; - /** - * Description to apply - */ - description?: string; - /** - * Owners to apply - * - * Custom Properties keys to remove - */ - customProperties?: any; - /** - * tier to apply - */ - tier?: TierElement; - /** - * Test Cases to apply - */ - testCases?: TestCaseDefinitions[]; - /** - * Remove all test cases - */ - removeAll?: boolean; - /** - * Test Cases to remove - */ - testCaseDefinitions?: string[]; - /** - * Owners to apply - */ - owners?: EntityReference[]; - /** - * Data Products to apply - * - * Data Products to remove - */ - dataProducts?: EntityReference[]; - /** - * Propagate the metadata to columns via column-level lineage. - */ - propagateColumnLevel?: boolean; - /** - * Propagate description through lineage - */ - propagateDescription?: boolean; - /** - * Propagate domains from the parent through lineage - */ - propagateDomains?: boolean; - /** - * Propagate glossary terms through lineage - */ - propagateGlossaryTerms?: boolean; - /** - * Propagate owner from the parent - */ - propagateOwner?: boolean; - /** - * Propagate the metadata to the parents (e.g., tables) via lineage. - */ - propagateParent?: boolean; - /** - * Propagate tags through lineage - */ - propagateTags?: boolean; - /** - * Propagate tier from the parent - */ - propagateTier?: boolean; - /** - * Number of levels to propagate lineage. If not set, it will propagate to all levels. - */ - propagationDepth?: number; - /** - * List of configurations to stop propagation based on conditions - */ - propagationStopConfigs?: PropagationStopConfig[]; -} - -/** - * Remove tags by its label type - * - * Remove terms by its label type - */ -export enum LabelElement { - Automated = "Automated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Configuration to stop lineage propagation based on conditions - */ -export interface PropagationStopConfig { - /** - * The metadata attribute to check for stopping propagation - */ - metadataAttribute: MetadataAttribute; - /** - * List of attribute values that will stop propagation when any of them is matched - */ - value: Array; -} - -/** - * The metadata attribute to check for stopping propagation - */ -export enum MetadataAttribute { - Description = "description", - Domains = "domains", - GlossaryTerms = "glossaryTerms", - Owner = "owner", - Tags = "tags", - Tier = "tier", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - * - * Owner of this Ingestion Pipeline. - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * The processing engine responsible for executing the ingestion pipeline logic. - * - * Link to the service for which ingestion pipeline is ingesting the metadata. - * - * Service to be modified - */ -export interface TagLabel { - /** - * Description for the tag label. - * - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this tag. - * - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Link to the tag resource. - * - * Link to the entity resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType?: LabelTypeEnum; - /** - * Name of the tag or glossary term. - * - * Name of the entity instance. - */ - name?: string; - /** - * Label is from Tags or Glossary. - */ - source?: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state?: State; - style?: Style; - tagFQN?: string; - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id?: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type?: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelTypeEnum { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - */ -export interface TierElement { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelTypeEnum; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Minimum set of requirements to get a Test Case request ready - */ -export interface TestCaseDefinitions { - /** - * Compute the passed and failed row count for the test case. - */ - computePassedFailedRowCount?: boolean; - parameterValues?: TestCaseParameterValue[]; - /** - * Tags to apply - */ - tags?: TierElement[]; - /** - * Fully qualified name of the test definition. - */ - testDefinition?: string; - /** - * If the test definition supports it, use dynamic assertion to evaluate the test case. - */ - useDynamicAssertion?: boolean; - [property: string]: any; -} - -/** - * This schema defines the parameter values that can be passed for a Test Case. - */ -export interface TestCaseParameterValue { - /** - * name of the parameter. Must match the parameter names in testCaseParameterDefinition - */ - name?: string; - /** - * value to be passed for the Parameters. These are input from Users. We capture this in - * string and convert during the runtime. - */ - value?: string; - [property: string]: any; -} - -/** - * Application Type - * - * Add Tags action type. - * - * Remove Classification Tags Action Type. - * - * Add Terms action type. - * - * Remove Terms Action Type. - * - * Add Domain Action Type. - * - * Remove Domain Action Type - * - * Add Description Action Type. - * - * Add Custom Properties Action Type. - * - * Remove Description Action Type - * - * Add Tier Action Type. - * - * Remove Tier Action Type - * - * Add Test Case Action Type. - * - * Remove Test Case Action Type - * - * Add Owner Action Type. - * - * Remove Owner Action Type - * - * Remove Custom Properties Action Type. - * - * Add Data Products Action Type. - * - * Remove Data Products Action Type. - * - * Lineage propagation action type. - * - * ML PII Tagging action type. - */ -export enum ActionType { - AddCustomPropertiesAction = "AddCustomPropertiesAction", - AddDataProductAction = "AddDataProductAction", - AddDescriptionAction = "AddDescriptionAction", - AddDomainAction = "AddDomainAction", - AddOwnerAction = "AddOwnerAction", - AddTagsAction = "AddTagsAction", - AddTermsAction = "AddTermsAction", - AddTestCaseAction = "AddTestCaseAction", - AddTierAction = "AddTierAction", - LineagePropagationAction = "LineagePropagationAction", - MLTaggingAction = "MLTaggingAction", - RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", - RemoveDataProductAction = "RemoveDataProductAction", - RemoveDescriptionAction = "RemoveDescriptionAction", - RemoveDomainAction = "RemoveDomainAction", - RemoveOwnerAction = "RemoveOwnerAction", - RemoveTagsAction = "RemoveTagsAction", - RemoveTermsAction = "RemoveTermsAction", - RemoveTestCaseAction = "RemoveTestCaseAction", - RemoveTierAction = "RemoveTierAction", -} - -/** - * Backfill Configuration - */ -export interface BackfillConfiguration { - /** - * Enable Backfill for the configured dates - */ - enabled?: boolean; - /** - * Date for which the backfill will end - */ - endDate?: Date; - /** - * Date from which to start the backfill - */ - startDate?: Date; - [property: string]: any; -} - -/** - * Different Module Configurations - */ -export interface ModuleConfiguration { - /** - * App Analytics Module configuration - */ - appAnalytics: AppAnalyticsConfig; - /** - * Cost Analysis Insights Module configuration - */ - costAnalysis: CostAnalysisConfig; - /** - * Data Assets Insights Module configuration - */ - dataAssets: DataAssetsConfig; - /** - * Data Quality Insights Module configuration - */ - dataQuality: DataQualityConfig; -} - -/** - * App Analytics Module configuration - */ -export interface AppAnalyticsConfig { - /** - * If Enabled, App Analytics insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Cost Analysis Insights Module configuration - */ -export interface CostAnalysisConfig { - /** - * If Enabled, Cost Analysis insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Data Assets Insights Module configuration - */ -export interface DataAssetsConfig { - /** - * If Enabled, Data Asset insights will be populated when the App runs. - */ - enabled: boolean; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Defines the number of days the Data Assets Insights information will be kept. After it - * they will be deleted. - */ - retention?: number; - serviceFilter?: ServiceFilter; -} - -export interface ServiceFilter { - serviceName?: string; - serviceType?: string; -} - -/** - * Data Quality Insights Module configuration - */ -export interface DataQualityConfig { - /** - * If Enabled, Data Quality insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Entities selected to run the automation. - */ -export interface Resource { - /** - * Filter JSON tree to be used for rendering the filters in the UI. This comes from - * Immutable Tree type of react-awesome-query-builder. - */ - filterJsonTree?: string; - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - queryFilter?: string; - /** - * Type of the entity. E.g., 'table', 'chart',... - */ - type?: string[]; - [property: string]: any; -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Application Type - * - * Application type. - */ -export enum CollateAIAppConfigType { - AutoPilotApplication = "AutoPilotApplication", - Automator = "Automator", - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - DataInsights = "DataInsights", - DataInsightsReport = "DataInsightsReport", - SearchIndexing = "SearchIndexing", -} - -/** - * Application private configuration - * - * Private Configuration for the CollateAI External Application. - */ -export interface PrivateConfig { - /** - * Collate Server public URL. WAII will use this information to interact with the server. - * E.g., https://sandbox.getcollate.io - */ - collateURL?: string; - /** - * Limits for the CollateAI Application. - */ - limits?: AppLimitsConfig; - /** - * WAII API Token - */ - token?: string; - /** - * WAII API host URL - */ - waiiInstance?: string; - [property: string]: any; -} - -/** - * Limits for the CollateAI Application. - * - * Private Configuration for the App Limits. - */ -export interface AppLimitsConfig { - /** - * The records of the limits. - */ - actions: { [key: string]: number }; - /** - * The start of this limit cycle. - */ - billingCycleStart: Date; -} - -/** - * Available sources to fetch DBT catalog and manifest files. - * - * dbt Cloud configuration. - * - * DBT Catalog, Manifest and Run Results file path config. - * - * DBT Catalog, Manifest and Run Results HTTP path configuration. - * - * DBT Catalog, Manifest and Run Results files in S3 bucket. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in GCS storage. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in Azure bucket. We will search for - * catalog.json, manifest.json and run_results.json. - */ -export interface DBTConfigurationSource { - /** - * dbt cloud account Id - */ - dbtCloudAccountId?: string; - /** - * dbt cloud account authentication token - */ - dbtCloudAuthToken?: string; - /** - * dbt cloud job id. - */ - dbtCloudJobId?: string; - /** - * In case of multiple projects in a dbt cloud account, specify the project's id from which - * you want to extract the dbt run artifacts - */ - dbtCloudProjectId?: string; - /** - * URL to connect to your dbt cloud instance. E.g., https://cloud.getdbt.com or - * https://emea.dbt.com/ - */ - dbtCloudUrl?: string; - /** - * dbt Configuration type - */ - dbtConfigType: DbtConfigType; - /** - * DBT catalog file path to extract dbt models with their column schemas. - */ - dbtCatalogFilePath?: string; - /** - * DBT manifest file path to extract dbt models and associate with tables. - */ - dbtManifestFilePath?: string; - /** - * DBT run results file path to extract the test results information. - */ - dbtRunResultsFilePath?: string; - /** - * DBT sources file path to extract the freshness test result. - */ - dbtSourcesFilePath?: string; - /** - * DBT catalog http file path to extract dbt models with their column schemas. - */ - dbtCatalogHttpPath?: string; - /** - * DBT manifest http file path to extract dbt models and associate with tables. - */ - dbtManifestHttpPath?: string; - /** - * DBT run results http file path to extract the test results information. - */ - dbtRunResultsHttpPath?: string; - /** - * DBT sources http file path to extract freshness test results information. - */ - dbtSourcesHttpPath?: string; - /** - * Details of the bucket where the dbt files are stored - */ - dbtPrefixConfig?: DBTPrefixConfig; - dbtSecurityConfig?: DbtSecurityConfigClass; -} - -/** - * dbt Configuration type - */ -export enum DbtConfigType { - Azure = "azure", - Cloud = "cloud", - Gcs = "gcs", - HTTP = "http", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the dbt files are stored - */ -export interface DBTPrefixConfig { - /** - * Name of the bucket where the dbt files are stored - */ - dbtBucketName?: string; - /** - * Path of the folder where the dbt files are stored - */ - dbtObjectPrefix?: string; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - * - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - */ -export interface DbtSecurityConfigClass { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Use incremental Metadata extraction after the first execution. This is commonly done by - * getting the changes from Audit tables on the supporting databases. - */ -export interface IncrementalMetadataExtractionConfiguration { - /** - * If True, enables Metadata Extraction to be incremental - */ - enabled: boolean; - /** - * Number os days to search back for a successful pipeline run. The timestamp of the last - * found successful pipeline run will be used as a base to search for updated entities. - */ - lookbackDays?: number; - /** - * Number of days to add to the last successful pipeline run timestamp to search for updated - * entities. - */ - safetyMarginDays?: number; -} - -/** - * Details required to generate Lineage - */ -export interface LineageInformation { - /** - * List of service path prefixes for lineage matching. Supported formats: DBServiceName, - * DBServiceName.DatabaseName, DBServiceName.DatabaseName.SchemaName, or - * DBServiceName.DatabaseName.SchemaName.TableName - */ - dbServicePrefixes?: string[]; - /** - * List of Database Service Names for creation of lineage - */ - dbServiceNames?: string[]; - /** - * List of Storage Service Names for creation of lineage - */ - storageServiceNames?: string[]; - [property: string]: any; -} - -/** - * Operation to be performed on the entity - */ -export interface Operation { - /** - * Entity to be modified - */ - entityLink: string; - /** - * The id of the operation - */ - id: string; - /** - * The configuration for the operation to be applied - */ - parameters: ReverseIngestionConfig; - /** - * Templated SQL command to be used for the operation. Context parameters will be populated - * based on the event type. - */ - SQLTemplate?: string; - /** - * Type of operation to perform - */ - type: OperationType; -} - -/** - * The configuration for the operation to be applied - * - * Configuration for updating descriptions - * - * Configuration for updating owners - * - * Configuration for updating tags - */ -export interface ReverseIngestionConfig { - /** - * New description of the service - */ - newDescription?: string; - /** - * Previous description of the service - */ - previousDescription?: string; - /** - * Added owners to be applied - */ - addedOwners?: EntityReference[]; - /** - * Removed owners from the entity - */ - removedOwners?: EntityReference[]; - /** - * Added tags to be applied - */ - addedTags?: TierElement[]; - /** - * Removed tags of the entity - */ - removedTags?: TierElement[]; -} - -/** - * Type of operation to perform - */ -export enum OperationType { - UpdateDescription = "UPDATE_DESCRIPTION", - UpdateOwner = "UPDATE_OWNER", - UpdateTags = "UPDATE_TAGS", -} - -/** - * Processing Engine Configuration. If not provided, the Native Engine will be used by - * default. - * - * Configuration for the native metadata ingestion engine - * - * This schema defines the configuration for a Spark Engine runner. - */ -export interface ProcessingEngine { - /** - * The type of the engine configuration - */ - type: ProcessingEngineType; - config?: Config; - /** - * Spark Connect Remote URL. - */ - remote?: string; -} - -export interface Config { - /** - * Additional Spark configuration properties as key-value pairs. - */ - extraConfig?: { [key: string]: any }; - /** - * Temporary path to store the data. - */ - tempPath?: string; - [property: string]: any; -} - -/** - * The type of the engine configuration - */ -export enum ProcessingEngineType { - Native = "Native", - Spark = "Spark", -} - -/** - * Type of Profile Sample (percentage or rows) - */ -export enum ProfileSampleType { - Percentage = "PERCENTAGE", - Rows = "ROWS", -} - -/** - * Type of Sampling Method (BERNOULLI or SYSTEM) - */ -export enum SamplingMethodType { - Bernoulli = "BERNOULLI", - System = "SYSTEM", -} - -/** - * Service connections available for the logical test suite. - */ -export interface ServiceConnections { - /** - * Connection configuration for the source. ex: mysql , tableau connection. - */ - serviceConnection: ServiceConnection; - serviceName: string; -} - -/** - * Connection configuration for the source. ex: mysql , tableau connection. - * - * Supported services - * - * API Service Connection. - * - * Dashboard Connection. - * - * Database Connection. - * - * Metadata Service Connection. - * - * Pipeline Connection. - * - * MlModel Connection. - * - * Storage Connection. - * - * search Connection. - * - * Security Connection. - * - * Drive Connection. - */ -export interface ServiceConnection { - config?: ConfigObject; -} - -/** - * REST Connection Config - * - * Looker Connection Config - * - * Metabase Connection Config - * - * PowerBI Connection Config - * - * PowerBIReportServer Connection Config - * - * Redash Connection Config - * - * Superset Connection Config - * - * Tableau Connection Config - * - * Mode Connection Config - * - * Custom Dashboard Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Domo Dashboard Connection Config - * - * QuickSight Connection Config - * - * Qlik Sense Connection Config - * - * Lightdash Connection Config - * - * MicroStrategy Connection Config - * - * Qlik Cloud Connection Config - * - * Sigma Connection Config - * - * ThoughtSpot Connection Config - * - * Grafana Connection Config - * - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - * - * Kafka Connection Config - * - * Redpanda Connection Config - * - * Kinesis Connection Config - * - * Custom Messaging Service Connection to build a source that is not supported by - * OpenMetadata yet. - * - * Amundsen Connection Config - * - * Metadata to ElasticSearch Connection Config - * - * OpenMetadata Connection Config - * - * Atlas Connection Config - * - * Alation Connection Config - * - * Alation Sink Connection Config - * - * Airflow Metadata Database Connection Config - * - * Wherescape Metadata Database Connection Config - * - * SSIS Metadata Database Connection Config - * - * Glue Pipeline Connection Config - * - * Airbyte Metadata Database Connection Config - * - * Fivetran Metadata Database Connection Config - * - * Flink Metadata Connection Config - * - * Dagster Metadata Database Connection Config - * - * Nifi Metadata Pipeline Connection Config - * - * Domo Pipeline Connection Config - * - * Custom Pipeline Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Spline Metadata Database Connection Config - * - * Spark Metadata Pipeline Connection Config - * - * OpenLineage Connection Config - * - * KafkaConnect Connection Config - * - * DBTCloud Connection Config - * - * Matillion Connection - * - * Azure Data Factory Connection Config - * - * Stitch Connection - * - * Snowplow Pipeline Connection Config - * - * MlFlow Connection Config - * - * Sklearn Connection Config - * - * Custom MlModel Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * SageMaker Connection Config - * - * Google VertexAI Connection Config - * - * S3 Connection. - * - * ADLS Connection. - * - * GCS Connection. - * - * Custom Storage Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * ElasticSearch Connection. - * - * OpenSearch Connection Config - * - * Custom Search Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * Apache Ranger Connection Config - * - * Google Drive Connection Config - * - * SharePoint Connection Config - * - * Custom Drive Connection to build a source that is not supported. - */ -export interface ConfigObject { - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Documentation URL for the schema. - */ - docURL?: string; - /** - * Open API Schema URL. - */ - openAPISchemaURL?: string; - /** - * Supports Metadata Extraction. - */ - supportsMetadataExtraction?: boolean; - /** - * Generated Token to connect to OpenAPI Schema. - * - * token to connect to Qlik Cloud. - * - * Generated Token to connect to Databricks. - * - * To Connect to Dagster Cloud - * - * Generated Token to connect to DBTCloud. - * - * Token to connect to Stitch api doc - */ - token?: string; - /** - * REST API Type - * - * Service Type - * - * Custom dashboard service type - * - * Custom database service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * OpenSearch Type - * - * Custom search service type - */ - type?: PurpleType; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * User's Client ID. This user should have privileges to read all the metadata in Looker. - * - * client_id for PowerBI. - * - * Client ID for DOMO - * - * client_id for Sigma. - * - * Azure Application (client) ID for service principal authentication. - * - * Application (client) ID from Azure Active Directory - */ - clientId?: string; - /** - * User's Client Secret. - * - * clientSecret for PowerBI. - * - * clientSecret for Sigma. - * - * Azure Application client secret for service principal authentication. - * - * Application (client) secret from Azure Active Directory - */ - clientSecret?: string; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - */ - gitCredentials?: GitHubCredentials; - /** - * URL to the Looker instance. - * - * Host and Port of the Metabase instance. - * - * Dashboard URL for PowerBI service. - * - * Dashboard URL for PowerBI Report Server. - * - * URL for the Redash instance - * - * URL for the superset instance. - * - * Tableau Server url. - * - * URL for the mode instance. - * - * URL for the Qlik instance. - * - * Address for your running Lightdash instance - * - * Host and Port of the MicroStrategy instance. - * - * Host and Port of the Qlik Cloud instance. - * - * Sigma API url. - * - * ThoughtSpot instance URL. Example: https://my-company.thoughtspot.cloud - * - * URL to the Grafana instance. - * - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - * - * Host and port of the Amundsen Neo4j Connection. This expect a URI format like: - * bolt://localhost:7687. - * - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - * - * Host and port of the Atlas service. - * - * Host and port of the Alation service. - * - * Pipeline Service Management/UI URI. - * - * Pipeline Service Management/UI URL. - * - * Spline REST Server Host & Port. - * - * KafkaConnect Service Management/UI URI. - * - * Host and port of the Stitch API host - * - * Host and port of the ElasticSearch service. - * - * Host and port of the OpenSearch service. - * - * Apache Ranger Admin URL. - */ - hostPort?: string; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * API token to connect to Metabase. Use this instead of username/password for token-based - * authentication. - * - * API key of the redash instance to access. - * - * The personal access token you can generate in the Lightdash app under the user settings - * - * Service Account Token to authenticate to the Grafana APIs. Use Service Account Tokens - * (format: glsa_xxxx) for authentication. Legacy API Keys are no longer supported by - * Grafana as of January 2025. Both self-hosted and Grafana Cloud are supported. Requires - * Admin role for full metadata extraction. - * - * API key to authenticate with the SAP ERP APIs. - * - * Fivetran API Secret. - * - * API Key for Snowplow Console API - */ - apiKey?: string; - /** - * Password to connect to Metabase. Required for basic authentication. - * - * Password to connect to PowerBI report server. - * - * Password to connect to MicroStrategy. - * - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to the Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - * - * password to connect to the Amundsen Neo4j Connection. - * - * password to connect to the Atlas. - * - * Password to connect to Airbyte. - */ - password?: string; - /** - * Username to connect to Metabase. Required for basic authentication. - * - * Username to connect to PowerBI report server. - * - * Username for Redash - * - * Username to connect to MicroStrategy. This user should have privileges to read all the - * metadata in MicroStrategy. - * - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to the Salesforce. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - * - * username to connect to the Amundsen Neo4j Connection. - * - * username to connect to the Atlas. This user should have privileges to read all the - * metadata in Atlas. - * - * Username to connect to Airbyte. - */ - username?: string; - /** - * Authority URI for the PowerBI service. - */ - authorityURI?: string; - /** - * Display Table Name from source instead of renamed table name for datamodel tables - */ - displayTableNameFromSource?: boolean; - /** - * Entity Limit set here will be used to paginate the PowerBi APIs - */ - pagination_entity_per_page?: number; - /** - * Source to get the .pbit files to extract lineage information - */ - pbitFilesSource?: PowerBIPbitFilesSource; - /** - * PowerBI secrets. - */ - scope?: string[]; - /** - * Tenant ID for PowerBI. - * - * Azure Directory (tenant) ID for service principal authentication. - * - * Directory (tenant) ID from Azure Active Directory - */ - tenantId?: string; - /** - * Fetch the PowerBI metadata using admin APIs - */ - useAdminApis?: boolean; - /** - * Web Portal Virtual Directory Name. - */ - webPortalVirtualDirectory?: string; - /** - * Version of the Redash instance - */ - redashVersion?: string; - /** - * Choose between API or database connection fetch metadata from superset. - * - * Choose between Database connection or HDB User Store connection. - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Matillion Auth Configuration - */ - connection?: ConfigConnection; - /** - * Tableau API version. If not provided, the version will be used from the tableau server. - * - * Sigma API version. - * - * ThoughtSpot API version to use - * - * OpenMetadata server API version to use. - * - * Airbyte API version. - */ - apiVersion?: string; - /** - * Types of methods used to authenticate to the tableau instance - * - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - * - * Types of methods used to authenticate to the alation instance - * - * Authentication type to connect to Apache Ranger. - */ - authType?: AuthenticationTypeForTableau | NoConfigAuthenticationTypes; - /** - * Pagination limit used while querying the tableau metadata API for getting data sources - * - * Pagination limit used while querying the SAP ERP API for fetching the entities - * - * Pagination limit used for Alation APIs pagination - */ - paginationLimit?: number; - /** - * Proxy URL for the tableau server. If not provided, the hostPort will be used. This is - * used to generate the dashboard & Chart URL. - */ - proxyURL?: string; - /** - * Tableau Site Name. - * - * SharePoint site name - */ - siteName?: string; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; - /** - * Boolean marking if we need to verify the SSL certs for Grafana. Default to True. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Boolean marking if we need to verify the SSL certs for KafkaConnect REST API. True by - * default. - */ - verifySSL?: boolean | VerifySSL; - /** - * Access Token for Mode Dashboard - * - * Access token to connect to DOMO - */ - accessToken?: string; - /** - * Access Token Password for Mode Dashboard - */ - accessTokenPassword?: string; - /** - * Filter query parameter for some of the Mode API calls - */ - filterQueryParam?: string; - /** - * Mode Workspace Name - */ - workspaceName?: string; - connectionOptions?: { [key: string]: string }; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - * - * Secret token to connect to DOMO - */ - secretToken?: string; - /** - * AWS Account ID - */ - awsAccountId?: string; - awsConfig?: AWSCredentials; - /** - * The authentication method that the user uses to sign in. - */ - identityType?: IdentityType; - /** - * The Amazon QuickSight namespace that contains the dashboard IDs in this request ( To be - * provided when identityType is `ANONYMOUS` ) - */ - namespace?: string; - certificates?: QlikCertificatesBy; - /** - * Qlik Sense Base URL, used for genrating dashboard & chat url - */ - displayUrl?: string; - /** - * User Directory. - */ - userDirectory?: string; - /** - * User ID. - */ - userId?: string; - /** - * Validate Host Name - */ - validateHostName?: boolean; - /** - * The Project UUID for your Lightdash instance - */ - projectUUID?: string; - /** - * Use if your Lightdash instance is behind a proxy like (Cloud IAP) - */ - proxyAuthentication?: string; - /** - * The Space UUID for your Lightdash instance - */ - spaceUUID?: string; - /** - * Login Mode for Microstrategy's REST API connection. You can authenticate with one of the - * following authentication modes: `Standard (1)`, `Anonymous (8)`. Default will be - * `Standard (1)`. If you're using demo account for Microstrategy, it will be needed to - * authenticate through loginMode `8`. - */ - loginMode?: string; - /** - * MicroStrategy Project Name - * - * Project name to create the refreshToken. Can be anything - */ - projectName?: string; - /** - * Space types of Qlik Cloud to filter the dashboards ingested into the platform. - */ - spaceTypes?: SpaceType[]; - /** - * ThoughtSpot authentication configuration - */ - authentication?: Authenticationation; - /** - * Org ID for multi-tenant ThoughtSpot instances. This is applicable for ThoughtSpot Cloud - * only. - */ - orgId?: string; - /** - * Page size for pagination in API requests. Default is 100. - */ - pageSize?: number; - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - * - * Azure Credentials - * - * GCP Credentials for Google Drive API - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - * - * Connection timeout in seconds. - */ - connectionTimeout?: number | number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - * - * Available sources to fetch metadata. - */ - configSource?: DeltaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - */ - auth?: AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - * - * Snowplow BDP Organization ID - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - /** - * basic.auth.user.info schema registry config property, Client HTTP credentials in the form - * of username:password. - */ - basicAuthUserInfo?: string; - /** - * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - * - * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - */ - bootstrapServers?: string; - /** - * Confluent Kafka Consumer Config. From - * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - * - * Confluent Redpanda Consumer Config - */ - consumerConfig?: { [key: string]: any }; - /** - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - */ - consumerConfigSSL?: ConsumerConfigSSLClass; - /** - * sasl.mechanism Consumer Config property - */ - saslMechanism?: SaslMechanismType; - /** - * sasl.password consumer config property - */ - saslPassword?: string; - /** - * sasl.username consumer config property - */ - saslUsername?: string; - /** - * Confluent Kafka Schema Registry Config. From - * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient - * - * Confluent Redpanda Schema Registry Config. - */ - schemaRegistryConfig?: { [key: string]: any }; - /** - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ - schemaRegistrySSL?: ConsumerConfigSSLClass; - /** - * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get - * topic schema from registry. - */ - schemaRegistryTopicSuffixName?: string; - /** - * Confluent Kafka Schema Registry URL. - * - * Confluent Redpanda Schema Registry URL. - */ - schemaRegistryURL?: string; - /** - * security.protocol consumer config property - * - * Kafka security protocol config - */ - securityProtocol?: KafkaSecurityProtocol; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Enable encryption for the Amundsen Neo4j Connection. - */ - encrypted?: boolean; - /** - * Maximum connection lifetime for the Amundsen Neo4j Connection. - */ - maxConnectionLifeTime?: number; - /** - * Enable SSL validation for the Amundsen Neo4j Connection. - */ - validateSSL?: boolean; - /** - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - /** - * List of entities that you need to reindex - */ - entities?: string[]; - recreateIndex?: boolean; - runMode?: RunMode; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * service type of the data source. - */ - databaseServiceName?: string[]; - /** - * Name of the Entity Type available in Atlas. - */ - entity_type?: string; - /** - * service type of the messaging source - * - * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. - * e.g. local_kafka - */ - messagingServiceName?: string[] | string; - /** - * Custom OpenMetadata Classification name for alation tags. - */ - alationTagClassificationName?: string; - /** - * Specifies if hidden datasources should be included while ingesting. - */ - includeHiddenDatasources?: boolean; - /** - * Specifies if undeployed datasources should be included while ingesting. - */ - includeUndeployedDatasources?: boolean; - /** - * Specifies if Dashboards are to be ingested while running the ingestion job. - */ - ingestDashboards?: boolean; - /** - * Specifies if Datasources are to be ingested while running the ingestion job. - */ - ingestDatasources?: boolean; - /** - * Specifies if Domains are to be ingested while running the ingestion job. - */ - ingestDomains?: boolean; - /** - * Specifies if Knowledge Articles are to be ingested while running the ingestion job. - */ - ingestKnowledgeArticles?: boolean; - /** - * Specifies if Users and Groups are to be ingested while running the ingestion job. - */ - ingestUsersAndGroups?: boolean; - datasourceLinks?: { [key: string]: string }; - /** - * Pipeline Service Number Of Status - */ - numberOfStatus?: number; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Underlying database connection - */ - databaseConnection?: DatabaseConnectionClass; - /** - * Underlying storage connection - */ - packageConnection?: S3Connection | string; - /** - * Fivetran API Secret. - */ - apiSecret?: string; - /** - * Fivetran API Limit For Pagination. - */ - limit?: number; - /** - * URL to the Dagster instance - * - * DBT cloud Access URL. - */ - host?: string; - /** - * Connection Time Limit Between OM and Dagster Graphql API in second - */ - timeout?: number; - /** - * We support username/password or client certificate authentication - */ - nifiConfig?: NifiCredentialsConfiguration; - /** - * Spline UI Host & Port. - */ - uiHostPort?: string; - /** - * service type of the messaging source - */ - brokersUrl?: string; - /** - * consumer group name - */ - consumerGroupName?: string; - /** - * initial Kafka consumer offset - */ - consumerOffsets?: InitialConsumerOffsets; - /** - * max allowed wait time - */ - poolTimeout?: number; - /** - * SASL Configuration details. - */ - saslConfig?: SASLClientConfig; - /** - * max allowed inactivity time - */ - sessionTimeout?: number; - /** - * topic from where Open lineage events will be pulled - */ - topicName?: string; - /** - * We support username/password or No Authentication - */ - KafkaConnectConfig?: UsernamePasswordAuthentication; - /** - * ID of your DBT cloud account - */ - accountId?: string; - /** - * DBT cloud Metadata API URL. - */ - discoveryAPI?: string; - /** - * List of IDs of your DBT cloud jobs seperated by comma `,` - */ - jobIds?: string[]; - /** - * Number of runs to fetch from DBT cloud - */ - numberOfRuns?: number; - /** - * List of IDs of your DBT cloud projects seperated by comma `,` - */ - projectIds?: string[]; - /** - * The name of your azure data factory. - */ - factory_name?: string; - /** - * The name of your resource group the data factory is associated with. - */ - resource_group_name?: string; - /** - * Number of days in the past to filter pipeline runs. - */ - run_filter_days?: number; - /** - * The azure subscription identifier. - */ - subscription_id?: string; - /** - * Cloud provider where Snowplow is deployed - */ - cloudProvider?: CloudProvider; - /** - * Path to pipeline configuration files for Community deployment - */ - configPath?: string; - /** - * Snowplow Console URL for BDP deployment - */ - consoleUrl?: string; - /** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - */ - deployment?: SnowplowDeployment; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Mlflow Model registry backend. E.g., - * mysql+pymysql://mlflow:password@localhost:3307/experiments - */ - registryUri?: string; - /** - * Mlflow Experiment tracking URI. E.g., http://localhost:5000 - */ - trackingUri?: string; - /** - * location/region of google cloud project - */ - location?: string; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Connection Timeout in Seconds - */ - connectionTimeoutSecs?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Email to impersonate using domain-wide delegation - */ - delegatedEmail?: string; - /** - * Specific shared drive ID to connect to - * - * SharePoint drive ID. If not provided, default document library will be used - */ - driveId?: string; - /** - * Extract metadata only for Google Sheets files - */ - includeGoogleSheets?: boolean; - /** - * Include shared/team drives in metadata extraction - */ - includeTeamDrives?: boolean; - /** - * SharePoint site URL - */ - siteUrl?: string; - [property: string]: any; -} - -/** - * We support username/password or No Authentication - * - * username/password auth - */ -export interface UsernamePasswordAuthentication { - /** - * KafkaConnect password to authenticate to the API. - */ - password?: string; - /** - * KafkaConnect user to authenticate to the API. - */ - username?: string; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Types of methods used to authenticate to the tableau instance - * - * Basic Auth Credentials - * - * Access Token Auth Credentials - * - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - * - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * API Access Token Auth Credentials - * - * Basic Auth Configuration for ElasticSearch - * - * SSL Certificates By Path - * - * AWS credentials configs. - * - * Authentication type to connect to Apache Ranger. - * - * Configuration for connecting to Ranger Basic Auth. - */ -export interface AuthenticationTypeForTableau { - /** - * Password to access the service. - * - * Password to connect to source. - * - * Elastic Search Password for Login - * - * Ranger password to authenticate to the API. - */ - password?: string; - /** - * Username to access the service. - * - * Elastic Search Username for Login - * - * Ranger user to authenticate to the API. - */ - username?: string; - /** - * Personal Access Token Name. - */ - personalAccessTokenName?: string; - /** - * Personal Access Token Secret. - */ - personalAccessTokenSecret?: string; - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; - /** - * Access Token for the API - */ - accessToken?: string; - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -/** - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * Basic Auth Credentials - * - * API Access Token Auth Credentials - */ -export interface Authenticationation { - /** - * Password to access the service. - */ - password?: string; - /** - * Username to access the service. - */ - username?: string; - /** - * Access Token for the API - */ - accessToken?: string; -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: Authentication; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum Authentication { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: Credentials | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Qlik Authentication Certificate By Values - * - * Qlik Authentication Certificate File Path - */ -export interface QlikCertificatesBy { - sslConfig?: ConsumerConfigSSLClass; - /** - * Client Certificate - */ - clientCertificate?: string; - /** - * Client Key Certificate. - */ - clientKeyCertificate?: string; - /** - * Root Certificate. - */ - rootCertificate?: string; - [property: string]: any; -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface ConsumerConfigSSLClass { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Cloud provider where Snowplow is deployed - */ -export enum CloudProvider { - Aws = "AWS", - Azure = "Azure", - Gcp = "GCP", -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface DeltaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConfigSourceConnection; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: DbtSecurityConfigClass; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConfigSourceConnection { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Choose between API or database connection fetch metadata from superset. - * - * Superset API Connection Config - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - * - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Lineage Backend Connection Config - * - * SQLite Database Connection Config - * - * Matillion Auth Configuration - * - * Matillion ETL Auth Config. - */ -export interface ConfigConnection { - /** - * Password for Superset. - * - * Password to connect to Hana. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to the Matillion. - */ - password?: string; - /** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ - provider?: Provider; - /** - * SSL Configuration details. - */ - sslConfig?: ConnectionSSLConfig; - /** - * Username for Superset. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to Hana. This user should have privileges to read all the metadata. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to the Matillion. This user should have privileges to read all the - * metadata in Matillion. - */ - username?: string; - verifySSL?: VerifySSL; - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - * - * Host and port of the Hana service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Matillion Host - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: ConnectionScheme; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: ConnectionType; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - supportsViewLineageExtraction?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ -export enum Provider { - DB = "db", - LDAP = "ldap", -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum ConnectionScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", - SqlitePysqlite = "sqlite+pysqlite", -} - -/** - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - */ -export interface ConnectionSSLConfig { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum ConnectionType { - Backend = "Backend", - MatillionETL = "MatillionETL", - Mysql = "Mysql", - Postgres = "Postgres", - SQLite = "SQLite", -} - -/** - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * initial Kafka consumer offset - */ -export enum InitialConsumerOffsets { - Earliest = "earliest", - Latest = "latest", -} - -/** - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Underlying database connection - * - * Mssql Database Connection Config - */ -export interface DatabaseConnectionClass { - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Host and port of the MSSQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to MSSQL. - */ - password?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: MssqlScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: MssqlType; - /** - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - */ - username?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum MssqlScheme { - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", -} - -/** - * Service Type - * - * Service type. - */ -export enum MssqlType { - Mssql = "Mssql", -} - -/** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - * - * Snowplow deployment type - */ -export enum SnowplowDeployment { - Bdp = "BDP", - Community = "Community", -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - * - * Do not set any credentials. Note that credentials are required to extract .lkml views and - * their lineage. - * - * Credentials for a GitHub repository - * - * Credentials for a BitBucket repository - * - * Credentials for a Gitlab repository - */ -export interface GitHubCredentials { - repositoryName?: string; - repositoryOwner?: string; - token?: string; - /** - * Credentials Type - */ - type?: GitHubCredentialsType; - /** - * Main production branch of the repository. E.g., `main` - */ - branch?: string; -} - -/** - * Credentials Type - * - * GitHub Credentials type - * - * BitBucket Credentials type - * - * Gitlab Credentials type - */ -export enum GitHubCredentialsType { - BitBucket = "BitBucket", - GitHub = "GitHub", - Gitlab = "Gitlab", -} - -/** - * The authentication method that the user uses to sign in. - */ -export enum IdentityType { - Anonymous = "ANONYMOUS", - Iam = "IAM", - Quicksight = "QUICKSIGHT", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: ConsumerConfigSSLClass; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * We support username/password or client certificate authentication - * - * Configuration for connecting to Nifi Basic Auth. - * - * Configuration for connecting to Nifi Client Certificate Auth. - */ -export interface NifiCredentialsConfiguration { - /** - * Nifi password to authenticate to the API. - */ - password?: string; - /** - * Nifi user to authenticate to the API. - */ - username?: string; - /** - * Boolean marking if we need to verify the SSL certs for Nifi. False by default. - */ - verifySSL?: boolean; - /** - * Path to the root CA certificate - */ - certificateAuthorityPath?: string; - /** - * Path to the client certificate - */ - clientCertificatePath?: string; - /** - * Path to the client key - */ - clientkeyPath?: string; -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * S3 Connection. - */ -export interface S3Connection { - awsConfig: AWSCredentials; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: S3Type; -} - -/** - * Service Type - * - * S3 service type - */ -export enum S3Type { - S3 = "S3", -} - -/** - * Source to get the .pbit files to extract lineage information - * - * Local config source where no extra information needs to be sent. - * - * Azure storage config for pbit files - * - * GCS storage config for pbit files - * - * S3 storage config for pbit files - */ -export interface PowerBIPbitFilesSource { - /** - * Directory path for the pbit files - */ - path?: string; - /** - * pbit File Configuration type - */ - pbitFileConfigType?: PbitFileConfigType; - /** - * Path of the folder where the .pbit files will be unzipped and datamodel schema will be - * extracted - */ - pbitFilesExtractDir?: string; - prefixConfig?: BucketDetails; - securityConfig?: DbtSecurityConfigClass; -} - -/** - * pbit File Configuration type - */ -export enum PbitFileConfigType { - Azure = "azure", - Gcs = "gcs", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the .pbit files are stored - */ -export interface BucketDetails { - /** - * Name of the bucket where the .pbit files are stored - */ - bucketName?: string; - /** - * Path of the folder where the .pbit files are stored - */ - objectPrefix?: string; -} - -/** - * This schema publisher run modes. - */ -export enum RunMode { - Batch = "batch", - Stream = "stream", -} - -/** - * SASL Configuration details. - * - * SASL client configuration. - */ -export interface SASLClientConfig { - /** - * SASL security mechanism - */ - saslMechanism?: SaslMechanismType; - /** - * The SASL authentication password. - */ - saslPassword?: string; - /** - * The SASL authentication username. - */ - saslUsername?: string; -} - -/** - * sasl.mechanism Consumer Config property - * - * SASL Mechanism consumer config property - * - * SASL security mechanism - */ -export enum SaslMechanismType { - Gssapi = "GSSAPI", - Oauthbearer = "OAUTHBEARER", - Plain = "PLAIN", - ScramSHA256 = "SCRAM-SHA-256", - ScramSHA512 = "SCRAM-SHA-512", -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * security.protocol consumer config property - * - * Kafka security protocol config - */ -export enum KafkaSecurityProtocol { - Plaintext = "PLAINTEXT", - SSL = "SSL", - SaslPlaintext = "SASL_PLAINTEXT", - SaslSSL = "SASL_SSL", -} - -export enum SpaceType { - Data = "Data", - Managed = "Managed", - Personal = "Personal", - Shared = "Shared", -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Config - */ -export interface SSLConfigObject { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; - /** - * SSL Certificates - */ - certificates?: SSLCertificates; - [property: string]: any; -} - -/** - * SSL Certificates - * - * SSL Configuration details. - * - * SSL Certificates By Path - * - * SSL Certificates By Values - */ -export interface SSLCertificates { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * CA Certificate Value - */ - caCertValue?: string; - /** - * Client Certificate Value - */ - clientCertValue?: string; - /** - * Private Key Value - */ - privateKeyValue?: string; - /** - * Staging Directory Path - */ - stagingDir?: string; -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * REST API Type - * - * REST API type - * - * Service Type - * - * Looker service type - * - * Metabase service type - * - * PowerBI service type - * - * PowerBIReportServer service type - * - * Redash service type - * - * Superset service type - * - * Tableau service type - * - * Mode service type - * - * Custom dashboard service type - * - * service type - * - * QuickSight service type - * - * Qlik sense service type - * - * Lightdash service type - * - * MicroStrategy service type - * - * Qlik Cloud service type - * - * Sigma service type - * - * ThoughtSpot service type - * - * Grafana service type - * - * Service type. - * - * Custom database service type - * - * Kafka service type - * - * Redpanda service type - * - * Custom messaging service type - * - * Amundsen service type - * - * Metadata to Elastic Search type - * - * OpenMetadata service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * S3 service type - * - * ADLS service type - * - * Gcs service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * ElasticSearch service type - * - * OpenSearch Type - * - * OpenSearch service type - * - * Custom search service type - * - * Apache Ranger service type - * - * Google Drive service type - * - * SharePoint service type - * - * Custom Drive service type - */ -export enum PurpleType { - Adls = "ADLS", - Airbyte = "Airbyte", - Airflow = "Airflow", - Alation = "Alation", - AlationSink = "AlationSink", - Amundsen = "Amundsen", - Athena = "Athena", - Atlas = "Atlas", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDashboard = "CustomDashboard", - CustomDatabase = "CustomDatabase", - CustomDrive = "CustomDrive", - CustomMessaging = "CustomMessaging", - CustomMlModel = "CustomMlModel", - CustomPipeline = "CustomPipeline", - CustomSearch = "CustomSearch", - CustomStorage = "CustomStorage", - DBTCloud = "DBTCloud", - Dagster = "Dagster", - DataFactory = "DataFactory", - Databricks = "Databricks", - DatabricksPipeline = "DatabricksPipeline", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDashboard = "DomoDashboard", - DomoDatabase = "DomoDatabase", - DomoPipeline = "DomoPipeline", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - ElasticSearch = "ElasticSearch", - Epic = "Epic", - Exasol = "Exasol", - Fivetran = "Fivetran", - Flink = "Flink", - Gcs = "GCS", - Glue = "Glue", - GluePipeline = "GluePipeline", - GoogleDrive = "GoogleDrive", - Grafana = "Grafana", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - Kafka = "Kafka", - KafkaConnect = "KafkaConnect", - Kinesis = "Kinesis", - Lightdash = "Lightdash", - Looker = "Looker", - MariaDB = "MariaDB", - Matillion = "Matillion", - Metabase = "Metabase", - MetadataES = "MetadataES", - MicroStrategy = "MicroStrategy", - Mlflow = "Mlflow", - Mode = "Mode", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Nifi = "Nifi", - OpenLineage = "OpenLineage", - OpenMetadata = "OpenMetadata", - OpenSearch = "OpenSearch", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - PowerBI = "PowerBI", - PowerBIReportServer = "PowerBIReportServer", - Presto = "Presto", - QlikCloud = "QlikCloud", - QlikSense = "QlikSense", - QuickSight = "QuickSight", - REST = "Rest", - Ranger = "Ranger", - Redash = "Redash", - Redpanda = "Redpanda", - Redshift = "Redshift", - S3 = "S3", - SAS = "SAS", - SQLite = "SQLite", - SageMaker = "SageMaker", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SharePoint = "SharePoint", - Sigma = "Sigma", - SingleStore = "SingleStore", - Sklearn = "Sklearn", - Snowflake = "Snowflake", - Snowplow = "Snowplow", - Spark = "Spark", - Spline = "Spline", - Ssas = "SSAS", - Ssis = "SSIS", - Stitch = "Stitch", - Superset = "Superset", - Synapse = "Synapse", - Tableau = "Tableau", - Teradata = "Teradata", - ThoughtSpot = "ThoughtSpot", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - VertexAI = "VertexAI", - Vertica = "Vertica", - Wherescape = "Wherescape", -} - -/** - * No manifest file available. Ingestion would look for bucket-level metadata file instead - * - * Storage Metadata Manifest file path config. - * - * Storage Metadata Manifest file HTTP path config. - * - * Storage Metadata Manifest file S3 path config. - * - * Storage Metadata Manifest file ADLS path config. - * - * Storage Metadata Manifest file GCS path config. - */ -export interface StorageMetadataConfigurationSource { - /** - * Storage Metadata manifest file path to extract locations to ingest from. - */ - manifestFilePath?: string; - /** - * Storage Metadata manifest http file path to extract locations to ingest from. - */ - manifestHttpPath?: string; - prefixConfig?: StorageMetadataBucketDetails; - securityConfig?: DbtSecurityConfigClass; -} - -/** - * Details of the bucket where the storage metadata manifest file is stored - */ -export interface StorageMetadataBucketDetails { - /** - * Name of the top level container where the storage metadata file is stored - */ - containerName: string; - /** - * Path of the folder where the storage metadata file is stored. If the file is at the root, - * you can keep it empty. - */ - objectPrefix?: string; -} - -/** - * Pipeline type - * - * Database Source Config Metadata Pipeline type - * - * Database Source Config Usage Pipeline type - * - * Dashboard Source Config Metadata Pipeline type - * - * Messaging Source Config Metadata Pipeline type - * - * Profiler Source Config Pipeline type - * - * Pipeline Source Config Metadata Pipeline type - * - * MlModel Source Config Metadata Pipeline type - * - * Object Store Source Config Metadata Pipeline type - * - * Drive Source Config Metadata Pipeline type - * - * Search Source Config Metadata Pipeline type - * - * DBT Config Pipeline type - * - * Pipeline Source Config For Application Pipeline type. Nothing is required. - * - * Api Source Config Metadata Pipeline type - * - * Reverse Ingestion Config Pipeline type - */ -export enum FluffyType { - APIMetadata = "ApiMetadata", - Application = "Application", - AutoClassification = "AutoClassification", - DashboardMetadata = "DashboardMetadata", - DataInsight = "dataInsight", - DatabaseLineage = "DatabaseLineage", - DatabaseMetadata = "DatabaseMetadata", - DatabaseUsage = "DatabaseUsage", - Dbt = "DBT", - DriveMetadata = "DriveMetadata", - MessagingMetadata = "MessagingMetadata", - MetadataToElasticSearch = "MetadataToElasticSearch", - MlModelMetadata = "MlModelMetadata", - PipelineMetadata = "PipelineMetadata", - Profiler = "Profiler", - ReverseIngestion = "ReverseIngestion", - SearchMetadata = "SearchMetadata", - StorageMetadata = "StorageMetadata", - TestSuite = "TestSuite", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts deleted file mode 100644 index 5a540d18c07..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts +++ /dev/null @@ -1,4153 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Test Service Connection to test user provided configuration is valid or not. - */ -export interface TestServiceConnection { - /** - * Connection object. - */ - connection?: TestServiceConnectionConnection; - /** - * Type of the connection to test such as Snowflake, MySQL, Looker, etc. - */ - connectionType?: string; - /** - * Optional value of the ingestion runner name responsible for running the test - */ - ingestionRunner?: string; - /** - * Secrets Manager Provider to use for fetching secrets. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * Optional value that identifies this service name. - */ - serviceName?: string; - /** - * Type of service such as Database, Dashboard, Messaging, etc. - */ - serviceType?: ServiceType; -} - -/** - * Connection object. - * - * API Service Connection. - * - * Database Connection. - * - * Dashboard Connection. - * - * Pipeline Connection. - * - * MlModel Connection. - * - * Metadata Service Connection. - * - * Storage Connection. - * - * search Connection. - * - * Security Connection. - * - * Drive Connection. - */ -export interface TestServiceConnectionConnection { - config?: ConfigObject; -} - -/** - * REST Connection Config - * - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - * - * Looker Connection Config - * - * Metabase Connection Config - * - * PowerBI Connection Config - * - * PowerBIReportServer Connection Config - * - * Redash Connection Config - * - * Superset Connection Config - * - * Tableau Connection Config - * - * Mode Connection Config - * - * Custom Dashboard Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Domo Dashboard Connection Config - * - * QuickSight Connection Config - * - * Qlik Sense Connection Config - * - * Lightdash Connection Config - * - * MicroStrategy Connection Config - * - * Qlik Cloud Connection Config - * - * Sigma Connection Config - * - * ThoughtSpot Connection Config - * - * Grafana Connection Config - * - * Kafka Connection Config - * - * Redpanda Connection Config - * - * Kinesis Connection Config - * - * Custom Messaging Service Connection to build a source that is not supported by - * OpenMetadata yet. - * - * Airflow Metadata Database Connection Config - * - * Wherescape Metadata Database Connection Config - * - * SSIS Metadata Database Connection Config - * - * Glue Pipeline Connection Config - * - * Airbyte Metadata Database Connection Config - * - * Fivetran Metadata Database Connection Config - * - * Flink Metadata Connection Config - * - * Dagster Metadata Database Connection Config - * - * Nifi Metadata Pipeline Connection Config - * - * Domo Pipeline Connection Config - * - * Custom Pipeline Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Spline Metadata Database Connection Config - * - * Spark Metadata Pipeline Connection Config - * - * OpenLineage Connection Config - * - * KafkaConnect Connection Config - * - * DBTCloud Connection Config - * - * Matillion Connection - * - * Azure Data Factory Connection Config - * - * Stitch Connection - * - * Snowplow Pipeline Connection Config - * - * MlFlow Connection Config - * - * Sklearn Connection Config - * - * Custom MlModel Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * SageMaker Connection Config - * - * Google VertexAI Connection Config - * - * Amundsen Connection Config - * - * Metadata to ElasticSearch Connection Config - * - * OpenMetadata Connection Config - * - * Atlas Connection Config - * - * Alation Connection Config - * - * Alation Sink Connection Config - * - * S3 Connection. - * - * ADLS Connection. - * - * GCS Connection. - * - * Custom Storage Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * ElasticSearch Connection. - * - * OpenSearch Connection Config - * - * Custom Search Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * Apache Ranger Connection Config - * - * Google Drive Connection Config - * - * SharePoint Connection Config - * - * Custom Drive Connection to build a source that is not supported. - */ -export interface ConfigObject { - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Documentation URL for the schema. - */ - docURL?: string; - /** - * Open API Schema URL. - */ - openAPISchemaURL?: string; - /** - * Supports Metadata Extraction. - */ - supportsMetadataExtraction?: boolean; - /** - * Generated Token to connect to OpenAPI Schema. - * - * Generated Token to connect to Databricks. - * - * token to connect to Qlik Cloud. - * - * To Connect to Dagster Cloud - * - * Generated Token to connect to DBTCloud. - * - * Token to connect to Stitch api doc - */ - token?: string; - /** - * REST API Type - * - * Service Type - * - * Custom database service type - * - * Custom dashboard service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * OpenSearch Type - * - * Custom search service type - */ - type?: ConfigType; - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - * - * Azure Credentials - * - * GCP Credentials for Google Drive API - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - * - * URL to the Looker instance. - * - * Host and Port of the Metabase instance. - * - * Dashboard URL for PowerBI service. - * - * Dashboard URL for PowerBI Report Server. - * - * URL for the Redash instance - * - * URL for the superset instance. - * - * Tableau Server url. - * - * URL for the mode instance. - * - * URL for the Qlik instance. - * - * Address for your running Lightdash instance - * - * Host and Port of the MicroStrategy instance. - * - * Host and Port of the Qlik Cloud instance. - * - * Sigma API url. - * - * ThoughtSpot instance URL. Example: https://my-company.thoughtspot.cloud - * - * URL to the Grafana instance. - * - * Pipeline Service Management/UI URI. - * - * Pipeline Service Management/UI URL. - * - * Spline REST Server Host & Port. - * - * KafkaConnect Service Management/UI URI. - * - * Host and port of the Stitch API host - * - * Host and port of the Amundsen Neo4j Connection. This expect a URI format like: - * bolt://localhost:7687. - * - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - * - * Host and port of the Atlas service. - * - * Host and port of the Alation service. - * - * Host and port of the ElasticSearch service. - * - * Host and port of the OpenSearch service. - * - * Apache Ranger Admin URL. - */ - hostPort?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - awsConfig?: AWSCredentials; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to the Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - * - * Password to connect to Metabase. Required for basic authentication. - * - * Password to connect to PowerBI report server. - * - * Password to connect to MicroStrategy. - * - * Password to connect to Airbyte. - * - * password to connect to the Amundsen Neo4j Connection. - * - * password to connect to the Atlas. - */ - password?: string; - /** - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to the Salesforce. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - * - * Username to connect to Metabase. Required for basic authentication. - * - * Username to connect to PowerBI report server. - * - * Username for Redash - * - * Username to connect to MicroStrategy. This user should have privileges to read all the - * metadata in MicroStrategy. - * - * Username to connect to Airbyte. - * - * username to connect to the Amundsen Neo4j Connection. - * - * username to connect to the Atlas. This user should have privileges to read all the - * metadata in Atlas. - */ - username?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - * - * Types of methods used to authenticate to the tableau instance - * - * Types of methods used to authenticate to the alation instance - * - * Authentication type to connect to Apache Ranger. - */ - authType?: AuthenticationType | NoConfigAuthenticationTypes; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - * - * Connection timeout in seconds. - */ - connectionTimeout?: number | number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - * - * Available sources to fetch metadata. - */ - configSource?: DeltaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - */ - auth?: AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - * - * Snowplow BDP Organization ID - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Access token to connect to DOMO - * - * Access Token for Mode Dashboard - */ - accessToken?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * Client ID for DOMO - * - * Azure Application (client) ID for service principal authentication. - * - * User's Client ID. This user should have privileges to read all the metadata in Looker. - * - * client_id for PowerBI. - * - * client_id for Sigma. - * - * Application (client) ID from Azure Active Directory - */ - clientId?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - * - * Secret token to connect to DOMO - */ - secretToken?: string; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * Choose between Database connection or HDB User Store connection. - * - * Choose between API or database connection fetch metadata from superset. - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Matillion Auth Configuration - * - * Choose between mysql and postgres connection for alation database - */ - connection?: ConfigConnection; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * API key to authenticate with the SAP ERP APIs. - * - * API token to connect to Metabase. Use this instead of username/password for token-based - * authentication. - * - * API key of the redash instance to access. - * - * The personal access token you can generate in the Lightdash app under the user settings - * - * Service Account Token to authenticate to the Grafana APIs. Use Service Account Tokens - * (format: glsa_xxxx) for authentication. Legacy API Keys are no longer supported by - * Grafana as of January 2025. Both self-hosted and Grafana Cloud are supported. Requires - * Admin role for full metadata extraction. - * - * Fivetran API Secret. - * - * API Key for Snowplow Console API - */ - apiKey?: string; - /** - * Pagination limit used while querying the SAP ERP API for fetching the entities - * - * Pagination limit used while querying the tableau metadata API for getting data sources - * - * Pagination limit used for Alation APIs pagination - */ - paginationLimit?: number; - /** - * Boolean marking if we need to verify the SSL certs for Grafana. Default to True. - * - * Boolean marking if we need to verify the SSL certs for KafkaConnect REST API. True by - * default. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - */ - verifySSL?: boolean | VerifySSL; - /** - * Azure Application client secret for service principal authentication. - * - * User's Client Secret. - * - * clientSecret for PowerBI. - * - * clientSecret for Sigma. - * - * Application (client) secret from Azure Active Directory - */ - clientSecret?: string; - /** - * Azure Directory (tenant) ID for service principal authentication. - * - * Tenant ID for PowerBI. - * - * Directory (tenant) ID from Azure Active Directory - */ - tenantId?: string; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - */ - gitCredentials?: GitHubCredentials; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * Authority URI for the PowerBI service. - */ - authorityURI?: string; - /** - * Display Table Name from source instead of renamed table name for datamodel tables - */ - displayTableNameFromSource?: boolean; - /** - * Entity Limit set here will be used to paginate the PowerBi APIs - */ - pagination_entity_per_page?: number; - /** - * Source to get the .pbit files to extract lineage information - */ - pbitFilesSource?: PowerBIPbitFilesSource; - /** - * PowerBI secrets. - */ - scope?: string[]; - /** - * Fetch the PowerBI metadata using admin APIs - */ - useAdminApis?: boolean; - /** - * Web Portal Virtual Directory Name. - */ - webPortalVirtualDirectory?: string; - /** - * Version of the Redash instance - */ - redashVersion?: string; - /** - * Tableau API version. If not provided, the version will be used from the tableau server. - * - * Sigma API version. - * - * ThoughtSpot API version to use - * - * Airbyte API version. - * - * OpenMetadata server API version to use. - */ - apiVersion?: string; - /** - * Proxy URL for the tableau server. If not provided, the hostPort will be used. This is - * used to generate the dashboard & Chart URL. - */ - proxyURL?: string; - /** - * Tableau Site Name. - * - * SharePoint site name - */ - siteName?: string; - /** - * Access Token Password for Mode Dashboard - */ - accessTokenPassword?: string; - /** - * Filter query parameter for some of the Mode API calls - */ - filterQueryParam?: string; - /** - * Mode Workspace Name - */ - workspaceName?: string; - /** - * AWS Account ID - */ - awsAccountId?: string; - /** - * The authentication method that the user uses to sign in. - */ - identityType?: IdentityType; - /** - * The Amazon QuickSight namespace that contains the dashboard IDs in this request ( To be - * provided when identityType is `ANONYMOUS` ) - */ - namespace?: string; - certificates?: QlikCertificatesBy; - /** - * Qlik Sense Base URL, used for genrating dashboard & chat url - */ - displayUrl?: string; - /** - * User Directory. - */ - userDirectory?: string; - /** - * User ID. - */ - userId?: string; - /** - * Validate Host Name - */ - validateHostName?: boolean; - /** - * The Project UUID for your Lightdash instance - */ - projectUUID?: string; - /** - * Use if your Lightdash instance is behind a proxy like (Cloud IAP) - */ - proxyAuthentication?: string; - /** - * The Space UUID for your Lightdash instance - */ - spaceUUID?: string; - /** - * Login Mode for Microstrategy's REST API connection. You can authenticate with one of the - * following authentication modes: `Standard (1)`, `Anonymous (8)`. Default will be - * `Standard (1)`. If you're using demo account for Microstrategy, it will be needed to - * authenticate through loginMode `8`. - */ - loginMode?: string; - /** - * MicroStrategy Project Name - * - * Project name to create the refreshToken. Can be anything - */ - projectName?: string; - /** - * Space types of Qlik Cloud to filter the dashboards ingested into the platform. - */ - spaceTypes?: SpaceType[]; - /** - * ThoughtSpot authentication configuration - */ - authentication?: Authenticationation; - /** - * Org ID for multi-tenant ThoughtSpot instances. This is applicable for ThoughtSpot Cloud - * only. - */ - orgId?: string; - /** - * Page size for pagination in API requests. Default is 100. - */ - pageSize?: number; - /** - * basic.auth.user.info schema registry config property, Client HTTP credentials in the form - * of username:password. - */ - basicAuthUserInfo?: string; - /** - * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - * - * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - */ - bootstrapServers?: string; - /** - * Confluent Kafka Consumer Config. From - * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - * - * Confluent Redpanda Consumer Config - */ - consumerConfig?: { [key: string]: any }; - /** - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - */ - consumerConfigSSL?: ConsumerConfigSSLClass; - /** - * sasl.mechanism Consumer Config property - */ - saslMechanism?: SaslMechanismType; - /** - * sasl.password consumer config property - */ - saslPassword?: string; - /** - * sasl.username consumer config property - */ - saslUsername?: string; - /** - * Confluent Kafka Schema Registry Config. From - * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient - * - * Confluent Redpanda Schema Registry Config. - */ - schemaRegistryConfig?: { [key: string]: any }; - /** - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ - schemaRegistrySSL?: ConsumerConfigSSLClass; - /** - * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get - * topic schema from registry. - */ - schemaRegistryTopicSuffixName?: string; - /** - * Confluent Kafka Schema Registry URL. - * - * Confluent Redpanda Schema Registry URL. - */ - schemaRegistryURL?: string; - /** - * security.protocol consumer config property - * - * Kafka security protocol config - */ - securityProtocol?: KafkaSecurityProtocol; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Pipeline Service Number Of Status - */ - numberOfStatus?: number; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Underlying database connection - */ - databaseConnection?: DatabaseConnectionClass; - /** - * Underlying storage connection - */ - packageConnection?: S3Connection | string; - /** - * Fivetran API Secret. - */ - apiSecret?: string; - /** - * Fivetran API Limit For Pagination. - */ - limit?: number; - /** - * URL to the Dagster instance - * - * DBT cloud Access URL. - */ - host?: string; - /** - * Connection Time Limit Between OM and Dagster Graphql API in second - */ - timeout?: number; - /** - * We support username/password or client certificate authentication - */ - nifiConfig?: NifiCredentialsConfiguration; - /** - * Spline UI Host & Port. - */ - uiHostPort?: string; - /** - * service type of the messaging source - */ - brokersUrl?: string; - /** - * consumer group name - */ - consumerGroupName?: string; - /** - * initial Kafka consumer offset - */ - consumerOffsets?: InitialConsumerOffsets; - /** - * max allowed wait time - */ - poolTimeout?: number; - /** - * SASL Configuration details. - */ - saslConfig?: SASLClientConfig; - /** - * max allowed inactivity time - */ - sessionTimeout?: number; - /** - * topic from where Open lineage events will be pulled - */ - topicName?: string; - /** - * We support username/password or No Authentication - */ - KafkaConnectConfig?: UsernamePasswordAuthentication; - /** - * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. - * e.g. local_kafka - * - * service type of the messaging source - */ - messagingServiceName?: string[] | string; - /** - * ID of your DBT cloud account - */ - accountId?: string; - /** - * DBT cloud Metadata API URL. - */ - discoveryAPI?: string; - /** - * List of IDs of your DBT cloud jobs seperated by comma `,` - */ - jobIds?: string[]; - /** - * Number of runs to fetch from DBT cloud - */ - numberOfRuns?: number; - /** - * List of IDs of your DBT cloud projects seperated by comma `,` - */ - projectIds?: string[]; - /** - * The name of your azure data factory. - */ - factory_name?: string; - /** - * The name of your resource group the data factory is associated with. - */ - resource_group_name?: string; - /** - * Number of days in the past to filter pipeline runs. - */ - run_filter_days?: number; - /** - * The azure subscription identifier. - */ - subscription_id?: string; - /** - * Cloud provider where Snowplow is deployed - */ - cloudProvider?: CloudProvider; - /** - * Path to pipeline configuration files for Community deployment - */ - configPath?: string; - /** - * Snowplow Console URL for BDP deployment - */ - consoleUrl?: string; - /** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - */ - deployment?: SnowplowDeployment; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Mlflow Model registry backend. E.g., - * mysql+pymysql://mlflow:password@localhost:3307/experiments - */ - registryUri?: string; - /** - * Mlflow Experiment tracking URI. E.g., http://localhost:5000 - */ - trackingUri?: string; - /** - * location/region of google cloud project - */ - location?: string; - /** - * Enable encryption for the Amundsen Neo4j Connection. - */ - encrypted?: boolean; - /** - * Maximum connection lifetime for the Amundsen Neo4j Connection. - */ - maxConnectionLifeTime?: number; - /** - * Enable SSL validation for the Amundsen Neo4j Connection. - */ - validateSSL?: boolean; - /** - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - /** - * List of entities that you need to reindex - */ - entities?: string[]; - recreateIndex?: boolean; - runMode?: RunMode; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * service type of the data source. - */ - databaseServiceName?: string[]; - /** - * Name of the Entity Type available in Atlas. - */ - entity_type?: string; - /** - * Custom OpenMetadata Classification name for alation tags. - */ - alationTagClassificationName?: string; - /** - * Specifies if hidden datasources should be included while ingesting. - */ - includeHiddenDatasources?: boolean; - /** - * Specifies if undeployed datasources should be included while ingesting. - */ - includeUndeployedDatasources?: boolean; - /** - * Specifies if Dashboards are to be ingested while running the ingestion job. - */ - ingestDashboards?: boolean; - /** - * Specifies if Datasources are to be ingested while running the ingestion job. - */ - ingestDatasources?: boolean; - /** - * Specifies if Domains are to be ingested while running the ingestion job. - */ - ingestDomains?: boolean; - /** - * Specifies if Knowledge Articles are to be ingested while running the ingestion job. - */ - ingestKnowledgeArticles?: boolean; - /** - * Specifies if Users and Groups are to be ingested while running the ingestion job. - */ - ingestUsersAndGroups?: boolean; - datasourceLinks?: { [key: string]: string }; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Connection Timeout in Seconds - */ - connectionTimeoutSecs?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Email to impersonate using domain-wide delegation - */ - delegatedEmail?: string; - /** - * Specific shared drive ID to connect to - * - * SharePoint drive ID. If not provided, default document library will be used - */ - driveId?: string; - /** - * Extract metadata only for Google Sheets files - */ - includeGoogleSheets?: boolean; - /** - * Include shared/team drives in metadata extraction - */ - includeTeamDrives?: boolean; - /** - * SharePoint site URL - */ - siteUrl?: string; - [property: string]: any; -} - -/** - * We support username/password or No Authentication - * - * username/password auth - */ -export interface UsernamePasswordAuthentication { - /** - * KafkaConnect password to authenticate to the API. - */ - password?: string; - /** - * KafkaConnect user to authenticate to the API. - */ - username?: string; -} - -/** - * Regex to only fetch api collections with names matching the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - * - * Regex exclude or include charts that matches the pattern. - * - * Regex to exclude or include dashboards that matches the pattern. - * - * Regex exclude or include data models that matches the pattern. - * - * Regex to exclude or include projects that matches the pattern. - * - * Regex to only fetch topics that matches the pattern. - * - * Regex exclude pipelines. - * - * Regex to only fetch containers that matches the pattern. - * - * Regex to only fetch MlModels with names matching the pattern. - * - * Regex to only fetch search indexes that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - * - * Types of methods used to authenticate to the tableau instance - * - * Basic Auth Credentials - * - * Access Token Auth Credentials - * - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * API Access Token Auth Credentials - * - * Basic Auth Configuration for ElasticSearch - * - * SSL Certificates By Path - * - * AWS credentials configs. - * - * Authentication type to connect to Apache Ranger. - * - * Configuration for connecting to Ranger Basic Auth. - */ -export interface AuthenticationType { - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - /** - * Password to connect to source. - * - * Password to access the service. - * - * Elastic Search Password for Login - * - * Ranger password to authenticate to the API. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; - /** - * Username to access the service. - * - * Elastic Search Username for Login - * - * Ranger user to authenticate to the API. - */ - username?: string; - /** - * Personal Access Token Name. - */ - personalAccessTokenName?: string; - /** - * Personal Access Token Secret. - */ - personalAccessTokenSecret?: string; - /** - * Access Token for the API - */ - accessToken?: string; - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -/** - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * Basic Auth Credentials - * - * API Access Token Auth Credentials - */ -export interface Authenticationation { - /** - * Password to access the service. - */ - password?: string; - /** - * Username to access the service. - */ - username?: string; - /** - * Access Token for the API - */ - accessToken?: string; -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: Authentication; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum Authentication { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: Credentials | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Qlik Authentication Certificate By Values - * - * Qlik Authentication Certificate File Path - */ -export interface QlikCertificatesBy { - sslConfig?: ConsumerConfigSSLClass; - /** - * Client Certificate - */ - clientCertificate?: string; - /** - * Client Key Certificate. - */ - clientKeyCertificate?: string; - /** - * Root Certificate. - */ - rootCertificate?: string; - [property: string]: any; -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface ConsumerConfigSSLClass { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Cloud provider where Snowplow is deployed - */ -export enum CloudProvider { - Aws = "AWS", - Azure = "Azure", - Gcp = "GCP", -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface DeltaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConfigSourceConnection; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: SecurityConfigClass; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConfigSourceConnection { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - * - * GCP Credentials - * - * GCP credentials configs. - * - * GCP Credentials for Google Drive API - * - * AWS credentials configs. - */ -export interface SecurityConfigClass { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - * - * Choose between API or database connection fetch metadata from superset. - * - * Superset API Connection Config - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Lineage Backend Connection Config - * - * SQLite Database Connection Config - * - * Matillion Auth Configuration - * - * Matillion ETL Auth Config. - * - * Choose between mysql and postgres connection for alation database - */ -export interface ConfigConnection { - /** - * Database of the data source. - * - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Host and port of the Hana service. - * - * Host and port of the source service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Matillion Host - */ - hostPort?: string; - /** - * Password to connect to Hana. - * - * Password for Superset. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to the Matillion. - */ - password?: string; - /** - * Username to connect to Hana. This user should have privileges to read all the metadata. - * - * Username for Superset. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to the Matillion. This user should have privileges to read all the - * metadata in Matillion. - */ - username?: string; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; - /** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ - provider?: Provider; - /** - * SSL Configuration details. - */ - sslConfig?: ConnectionSSLConfig; - verifySSL?: VerifySSL; - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: ConnectionScheme; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: ConnectionType; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - supportsViewLineageExtraction?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ -export enum Provider { - DB = "db", - LDAP = "ldap", -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum ConnectionScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", - SqlitePysqlite = "sqlite+pysqlite", -} - -/** - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - */ -export interface ConnectionSSLConfig { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum ConnectionType { - Backend = "Backend", - MatillionETL = "MatillionETL", - Mysql = "Mysql", - Postgres = "Postgres", - SQLite = "SQLite", -} - -/** - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * initial Kafka consumer offset - */ -export enum InitialConsumerOffsets { - Earliest = "earliest", - Latest = "latest", -} - -/** - * GCP Credentials - * - * GCP credentials configs. - * - * GCP Credentials for Google Drive API - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Underlying database connection - * - * Mssql Database Connection Config - */ -export interface DatabaseConnectionClass { - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Host and port of the MSSQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to MSSQL. - */ - password?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: MssqlScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: MssqlType; - /** - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - */ - username?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum MssqlScheme { - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", -} - -/** - * Service Type - * - * Service type. - */ -export enum MssqlType { - Mssql = "Mssql", -} - -/** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - * - * Snowplow deployment type - */ -export enum SnowplowDeployment { - Bdp = "BDP", - Community = "Community", -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - * - * Do not set any credentials. Note that credentials are required to extract .lkml views and - * their lineage. - * - * Credentials for a GitHub repository - * - * Credentials for a BitBucket repository - * - * Credentials for a Gitlab repository - */ -export interface GitHubCredentials { - repositoryName?: string; - repositoryOwner?: string; - token?: string; - /** - * Credentials Type - */ - type?: GitHubCredentialsType; - /** - * Main production branch of the repository. E.g., `main` - */ - branch?: string; -} - -/** - * Credentials Type - * - * GitHub Credentials type - * - * BitBucket Credentials type - * - * Gitlab Credentials type - */ -export enum GitHubCredentialsType { - BitBucket = "BitBucket", - GitHub = "GitHub", - Gitlab = "Gitlab", -} - -/** - * The authentication method that the user uses to sign in. - */ -export enum IdentityType { - Anonymous = "ANONYMOUS", - Iam = "IAM", - Quicksight = "QUICKSIGHT", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: ConsumerConfigSSLClass; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * We support username/password or client certificate authentication - * - * Configuration for connecting to Nifi Basic Auth. - * - * Configuration for connecting to Nifi Client Certificate Auth. - */ -export interface NifiCredentialsConfiguration { - /** - * Nifi password to authenticate to the API. - */ - password?: string; - /** - * Nifi user to authenticate to the API. - */ - username?: string; - /** - * Boolean marking if we need to verify the SSL certs for Nifi. False by default. - */ - verifySSL?: boolean; - /** - * Path to the root CA certificate - */ - certificateAuthorityPath?: string; - /** - * Path to the client certificate - */ - clientCertificatePath?: string; - /** - * Path to the client key - */ - clientkeyPath?: string; -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * S3 Connection. - */ -export interface S3Connection { - awsConfig: AWSCredentials; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: S3Type; -} - -/** - * Service Type - * - * S3 service type - */ -export enum S3Type { - S3 = "S3", -} - -/** - * Source to get the .pbit files to extract lineage information - * - * Local config source where no extra information needs to be sent. - * - * Azure storage config for pbit files - * - * GCS storage config for pbit files - * - * S3 storage config for pbit files - */ -export interface PowerBIPbitFilesSource { - /** - * Directory path for the pbit files - */ - path?: string; - /** - * pbit File Configuration type - */ - pbitFileConfigType?: PbitFileConfigType; - /** - * Path of the folder where the .pbit files will be unzipped and datamodel schema will be - * extracted - */ - pbitFilesExtractDir?: string; - prefixConfig?: BucketDetails; - securityConfig?: SecurityConfigClass; -} - -/** - * pbit File Configuration type - */ -export enum PbitFileConfigType { - Azure = "azure", - Gcs = "gcs", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the .pbit files are stored - */ -export interface BucketDetails { - /** - * Name of the bucket where the .pbit files are stored - */ - bucketName?: string; - /** - * Path of the folder where the .pbit files are stored - */ - objectPrefix?: string; -} - -/** - * This schema publisher run modes. - */ -export enum RunMode { - Batch = "batch", - Stream = "stream", -} - -/** - * SASL Configuration details. - * - * SASL client configuration. - */ -export interface SASLClientConfig { - /** - * SASL security mechanism - */ - saslMechanism?: SaslMechanismType; - /** - * The SASL authentication password. - */ - saslPassword?: string; - /** - * The SASL authentication username. - */ - saslUsername?: string; -} - -/** - * sasl.mechanism Consumer Config property - * - * SASL Mechanism consumer config property - * - * SASL security mechanism - */ -export enum SaslMechanismType { - Gssapi = "GSSAPI", - Oauthbearer = "OAUTHBEARER", - Plain = "PLAIN", - ScramSHA256 = "SCRAM-SHA-256", - ScramSHA512 = "SCRAM-SHA-512", -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - * - * Secrets Manager Provider to use for fetching secrets. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * security.protocol consumer config property - * - * Kafka security protocol config - */ -export enum KafkaSecurityProtocol { - Plaintext = "PLAINTEXT", - SSL = "SSL", - SaslPlaintext = "SASL_PLAINTEXT", - SaslSSL = "SASL_SSL", -} - -export enum SpaceType { - Data = "Data", - Managed = "Managed", - Personal = "Personal", - Shared = "Shared", -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Config - */ -export interface SSLConfigObject { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; - /** - * SSL Certificates - */ - certificates?: SSLCertificates; - [property: string]: any; -} - -/** - * SSL Certificates - * - * SSL Configuration details. - * - * SSL Certificates By Path - * - * SSL Certificates By Values - */ -export interface SSLCertificates { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * CA Certificate Value - */ - caCertValue?: string; - /** - * Client Certificate Value - */ - clientCertValue?: string; - /** - * Private Key Value - */ - privateKeyValue?: string; - /** - * Staging Directory Path - */ - stagingDir?: string; -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * REST API Type - * - * REST API type - * - * Service Type - * - * Service type. - * - * service type - * - * Custom database service type - * - * Looker service type - * - * Metabase service type - * - * PowerBI service type - * - * PowerBIReportServer service type - * - * Redash service type - * - * Superset service type - * - * Tableau service type - * - * Mode service type - * - * Custom dashboard service type - * - * QuickSight service type - * - * Qlik sense service type - * - * Lightdash service type - * - * MicroStrategy service type - * - * Qlik Cloud service type - * - * Sigma service type - * - * ThoughtSpot service type - * - * Grafana service type - * - * Kafka service type - * - * Redpanda service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Amundsen service type - * - * Metadata to Elastic Search type - * - * OpenMetadata service type - * - * S3 service type - * - * ADLS service type - * - * Gcs service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * ElasticSearch service type - * - * OpenSearch Type - * - * OpenSearch service type - * - * Custom search service type - * - * Apache Ranger service type - * - * Google Drive service type - * - * SharePoint service type - * - * Custom Drive service type - */ -export enum ConfigType { - Adls = "ADLS", - Airbyte = "Airbyte", - Airflow = "Airflow", - Alation = "Alation", - AlationSink = "AlationSink", - Amundsen = "Amundsen", - Athena = "Athena", - Atlas = "Atlas", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDashboard = "CustomDashboard", - CustomDatabase = "CustomDatabase", - CustomDrive = "CustomDrive", - CustomMessaging = "CustomMessaging", - CustomMlModel = "CustomMlModel", - CustomPipeline = "CustomPipeline", - CustomSearch = "CustomSearch", - CustomStorage = "CustomStorage", - DBTCloud = "DBTCloud", - Dagster = "Dagster", - DataFactory = "DataFactory", - Databricks = "Databricks", - DatabricksPipeline = "DatabricksPipeline", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDashboard = "DomoDashboard", - DomoDatabase = "DomoDatabase", - DomoPipeline = "DomoPipeline", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - ElasticSearch = "ElasticSearch", - Epic = "Epic", - Exasol = "Exasol", - Fivetran = "Fivetran", - Flink = "Flink", - Gcs = "GCS", - Glue = "Glue", - GluePipeline = "GluePipeline", - GoogleDrive = "GoogleDrive", - Grafana = "Grafana", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - Kafka = "Kafka", - KafkaConnect = "KafkaConnect", - Kinesis = "Kinesis", - Lightdash = "Lightdash", - Looker = "Looker", - MariaDB = "MariaDB", - Matillion = "Matillion", - Metabase = "Metabase", - MetadataES = "MetadataES", - MicroStrategy = "MicroStrategy", - Mlflow = "Mlflow", - Mode = "Mode", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Nifi = "Nifi", - OpenLineage = "OpenLineage", - OpenMetadata = "OpenMetadata", - OpenSearch = "OpenSearch", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - PowerBI = "PowerBI", - PowerBIReportServer = "PowerBIReportServer", - Presto = "Presto", - QlikCloud = "QlikCloud", - QlikSense = "QlikSense", - QuickSight = "QuickSight", - REST = "Rest", - Ranger = "Ranger", - Redash = "Redash", - Redpanda = "Redpanda", - Redshift = "Redshift", - S3 = "S3", - SAS = "SAS", - SQLite = "SQLite", - SageMaker = "SageMaker", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SharePoint = "SharePoint", - Sigma = "Sigma", - SingleStore = "SingleStore", - Sklearn = "Sklearn", - Snowflake = "Snowflake", - Snowplow = "Snowplow", - Spark = "Spark", - Spline = "Spline", - Ssas = "SSAS", - Ssis = "SSIS", - Stitch = "Stitch", - Superset = "Superset", - Synapse = "Synapse", - Tableau = "Tableau", - Teradata = "Teradata", - ThoughtSpot = "ThoughtSpot", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - VertexAI = "VertexAI", - Vertica = "Vertica", - Wherescape = "Wherescape", -} - -/** - * Type of service such as Database, Dashboard, Messaging, etc. - * - * This schema defines the service types entities which requires a connection. - */ -export enum ServiceType { - API = "Api", - Dashboard = "Dashboard", - Database = "Database", - Drive = "Drive", - Messaging = "Messaging", - Metadata = "Metadata", - MlModel = "MlModel", - Pipeline = "Pipeline", - Search = "Search", - Security = "Security", - Storage = "Storage", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts deleted file mode 100644 index c2d6b1e7c1e..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts +++ /dev/null @@ -1,4908 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * A unit of work that will be triggered as an API call to the OpenMetadata server. - */ -export interface Workflow { - /** - * Change that lead to this version of the entity. - */ - changeDescription?: ChangeDescription; - /** - * List of data products this entity is part of. - */ - dataProducts?: EntityReference[]; - /** - * When `true` indicates the entity has been soft deleted. - */ - deleted?: boolean; - /** - * Description of the test connection def. - */ - description?: string; - /** - * Display Name that identifies this workflow definition. - */ - displayName?: string; - /** - * Domains the asset belongs to. When not set, the asset inherits the domain from the parent - * it belongs to. - */ - domains?: EntityReference[]; - /** - * FullyQualifiedName same as `name`. - */ - fullyQualifiedName?: string; - /** - * Link to the resource corresponding to this entity. - */ - href?: string; - /** - * Unique identifier of this workflow instance. - */ - id: string; - /** - * Change that lead to this version of the entity. - */ - incrementalChangeDescription?: ChangeDescription; - /** - * Name of the workflow. - */ - name: string; - openMetadataServerConnection?: OpenMetadataConnection; - /** - * Owners of this workflow. - */ - owners?: EntityReference[]; - /** - * Request body for a specific workflow type - */ - request: TestServiceConnectionRequest; - /** - * Response to the request. - */ - response?: TestConnectionResult; - /** - * Workflow computation status. - */ - status?: WorkflowStatus; - /** - * Last update time corresponding to the new version of the entity in Unix epoch time - * milliseconds. - */ - updatedAt?: number; - /** - * User who made the update. - */ - updatedBy?: string; - /** - * Metadata version of the entity. - */ - version?: number; - /** - * Type of the workflow. - */ - workflowType: WorkflowType; -} - -/** - * Change that lead to this version of the entity. - * - * Description of the change. - */ -export interface ChangeDescription { - changeSummary?: { [key: string]: ChangeSummary }; - /** - * Names of fields added during the version changes. - */ - fieldsAdded?: FieldChange[]; - /** - * Fields deleted during the version changes with old value before deleted. - */ - fieldsDeleted?: FieldChange[]; - /** - * Fields modified during the version changes with old and new values. - */ - fieldsUpdated?: FieldChange[]; - /** - * When a change did not result in change, this could be same as the current version. - */ - previousVersion?: number; -} - -export interface ChangeSummary { - changedAt?: number; - /** - * Name of the user or bot who made this change - */ - changedBy?: string; - changeSource?: ChangeSource; - [property: string]: any; -} - -/** - * The source of the change. This will change based on the context of the change (example: - * manual vs programmatic) - */ -export enum ChangeSource { - Automated = "Automated", - Derived = "Derived", - Ingested = "Ingested", - Manual = "Manual", - Propagated = "Propagated", - Suggested = "Suggested", -} - -export interface FieldChange { - /** - * Name of the entity field that changed. - */ - name?: string; - /** - * New value of the field. Note that this is a JSON string and use the corresponding field - * type to deserialize it. - */ - newValue?: any; - /** - * Previous value of the field. Note that this is a JSON string and use the corresponding - * field type to deserialize it. - */ - oldValue?: any; -} - -/** - * List of data products this entity is part of. - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Service to be modified - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * OpenMetadata Connection Config - */ -export interface OpenMetadataConnection { - /** - * OpenMetadata server API version to use. - */ - apiVersion?: string; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: OpenMetadataServerConnectionElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - */ - hostPort: string; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: ConsumerConfigSSLClass; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: OpenmetadataType; - /** - * Flag to verify SSL Certificate for OpenMetadata Server. - */ - verifySSL?: VerifySSL; -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to only fetch api collections with names matching the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - * - * Regex exclude or include charts that matches the pattern. - * - * Regex to exclude or include dashboards that matches the pattern. - * - * Regex exclude or include data models that matches the pattern. - * - * Regex to exclude or include projects that matches the pattern. - * - * Regex to only fetch topics that matches the pattern. - * - * Regex exclude pipelines. - * - * Regex to only fetch containers that matches the pattern. - * - * Regex to only fetch MlModels with names matching the pattern. - * - * Regex to only fetch search indexes that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface OpenMetadataServerConnectionElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - * - * Secrets Manager Provider to use for fetching secrets. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * SSL Configuration for OpenMetadata Server - * - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface ConsumerConfigSSLClass { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Service Type - * - * OpenMetadata service type - */ -export enum OpenmetadataType { - OpenMetadata = "OpenMetadata", -} - -/** - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * Request body for a specific workflow type - * - * Test Service Connection to test user provided configuration is valid or not. - * - * Apply a set of operations on a service - * - * Query Runner Request - * - * Test Spark Engine Connection to test user provided configuration is valid or not. - */ -export interface TestServiceConnectionRequest { - /** - * Connection object. - */ - connection?: RequestConnection; - /** - * Type of the connection to test such as Snowflake, MySQL, Looker, etc. - */ - connectionType?: string; - /** - * Optional value of the ingestion runner name responsible for running the test - * - * Optional value of the ingestion runner name responsible for running the workflow - */ - ingestionRunner?: string; - /** - * Secrets Manager Provider to use for fetching secrets. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * Optional value that identifies this service name. - */ - serviceName?: string; - /** - * Type of service such as Database, Dashboard, Messaging, etc. - */ - serviceType?: ServiceType; - /** - * List of operations to be performed on the service - */ - operations?: Operation[]; - /** - * Service to be modified - */ - service?: EntityReference; - /** - * Pipeline type - */ - type?: ReverseIngestionType; - /** - * Query to be executed. - */ - query?: string; - /** - * Optional value to indicate if the query should be transpiled. - */ - transpile?: boolean; - /** - * Spark Engine Configuration. - */ - sparkEngine?: SparkEngineConfiguration; -} - -/** - * Connection object. - * - * API Service Connection. - * - * Database Connection. - * - * Dashboard Connection. - * - * Pipeline Connection. - * - * MlModel Connection. - * - * Metadata Service Connection. - * - * Storage Connection. - * - * search Connection. - * - * Security Connection. - * - * Drive Connection. - */ -export interface RequestConnection { - config?: ConfigObject; -} - -/** - * REST Connection Config - * - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - * - * Looker Connection Config - * - * Metabase Connection Config - * - * PowerBI Connection Config - * - * PowerBIReportServer Connection Config - * - * Redash Connection Config - * - * Superset Connection Config - * - * Tableau Connection Config - * - * Mode Connection Config - * - * Custom Dashboard Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Domo Dashboard Connection Config - * - * QuickSight Connection Config - * - * Qlik Sense Connection Config - * - * Lightdash Connection Config - * - * MicroStrategy Connection Config - * - * Qlik Cloud Connection Config - * - * Sigma Connection Config - * - * ThoughtSpot Connection Config - * - * Grafana Connection Config - * - * Kafka Connection Config - * - * Redpanda Connection Config - * - * Kinesis Connection Config - * - * Custom Messaging Service Connection to build a source that is not supported by - * OpenMetadata yet. - * - * Airflow Metadata Database Connection Config - * - * Wherescape Metadata Database Connection Config - * - * SSIS Metadata Database Connection Config - * - * Glue Pipeline Connection Config - * - * Airbyte Metadata Database Connection Config - * - * Fivetran Metadata Database Connection Config - * - * Flink Metadata Connection Config - * - * Dagster Metadata Database Connection Config - * - * Nifi Metadata Pipeline Connection Config - * - * Domo Pipeline Connection Config - * - * Custom Pipeline Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Spline Metadata Database Connection Config - * - * Spark Metadata Pipeline Connection Config - * - * OpenLineage Connection Config - * - * KafkaConnect Connection Config - * - * DBTCloud Connection Config - * - * Matillion Connection - * - * Azure Data Factory Connection Config - * - * Stitch Connection - * - * Snowplow Pipeline Connection Config - * - * MlFlow Connection Config - * - * Sklearn Connection Config - * - * Custom MlModel Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * SageMaker Connection Config - * - * Google VertexAI Connection Config - * - * Amundsen Connection Config - * - * Metadata to ElasticSearch Connection Config - * - * OpenMetadata Connection Config - * - * Atlas Connection Config - * - * Alation Connection Config - * - * Alation Sink Connection Config - * - * S3 Connection. - * - * ADLS Connection. - * - * GCS Connection. - * - * Custom Storage Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * ElasticSearch Connection. - * - * OpenSearch Connection Config - * - * Custom Search Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * Apache Ranger Connection Config - * - * Google Drive Connection Config - * - * SharePoint Connection Config - * - * Custom Drive Connection to build a source that is not supported. - */ -export interface ConfigObject { - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Documentation URL for the schema. - */ - docURL?: string; - /** - * Open API Schema URL. - */ - openAPISchemaURL?: string; - /** - * Supports Metadata Extraction. - */ - supportsMetadataExtraction?: boolean; - /** - * Generated Token to connect to OpenAPI Schema. - * - * Generated Token to connect to Databricks. - * - * token to connect to Qlik Cloud. - * - * To Connect to Dagster Cloud - * - * Generated Token to connect to DBTCloud. - * - * Token to connect to Stitch api doc - */ - token?: string; - /** - * REST API Type - * - * Service Type - * - * Custom database service type - * - * Custom dashboard service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * OpenSearch Type - * - * Custom search service type - */ - type?: ConfigType; - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - * - * Azure Credentials - * - * GCP Credentials for Google Drive API - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - * - * URL to the Looker instance. - * - * Host and Port of the Metabase instance. - * - * Dashboard URL for PowerBI service. - * - * Dashboard URL for PowerBI Report Server. - * - * URL for the Redash instance - * - * URL for the superset instance. - * - * Tableau Server url. - * - * URL for the mode instance. - * - * URL for the Qlik instance. - * - * Address for your running Lightdash instance - * - * Host and Port of the MicroStrategy instance. - * - * Host and Port of the Qlik Cloud instance. - * - * Sigma API url. - * - * ThoughtSpot instance URL. Example: https://my-company.thoughtspot.cloud - * - * URL to the Grafana instance. - * - * Pipeline Service Management/UI URI. - * - * Pipeline Service Management/UI URL. - * - * Spline REST Server Host & Port. - * - * KafkaConnect Service Management/UI URI. - * - * Host and port of the Stitch API host - * - * Host and port of the Amundsen Neo4j Connection. This expect a URI format like: - * bolt://localhost:7687. - * - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - * - * Host and port of the Atlas service. - * - * Host and port of the Alation service. - * - * Host and port of the ElasticSearch service. - * - * Host and port of the OpenSearch service. - * - * Apache Ranger Admin URL. - */ - hostPort?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - awsConfig?: AWSCredentials; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to the Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - * - * Password to connect to Metabase. Required for basic authentication. - * - * Password to connect to PowerBI report server. - * - * Password to connect to MicroStrategy. - * - * Password to connect to Airbyte. - * - * password to connect to the Amundsen Neo4j Connection. - * - * password to connect to the Atlas. - */ - password?: string; - /** - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to the Salesforce. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - * - * Username to connect to Metabase. Required for basic authentication. - * - * Username to connect to PowerBI report server. - * - * Username for Redash - * - * Username to connect to MicroStrategy. This user should have privileges to read all the - * metadata in MicroStrategy. - * - * Username to connect to Airbyte. - * - * username to connect to the Amundsen Neo4j Connection. - * - * username to connect to the Atlas. This user should have privileges to read all the - * metadata in Atlas. - */ - username?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - * - * Types of methods used to authenticate to the tableau instance - * - * Types of methods used to authenticate to the alation instance - * - * Authentication type to connect to Apache Ranger. - */ - authType?: AuthenticationType | NoConfigAuthenticationTypes; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - * - * Connection timeout in seconds. - */ - connectionTimeout?: number | number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - * - * Available sources to fetch metadata. - */ - configSource?: DeltaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - */ - auth?: AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - * - * Snowplow BDP Organization ID - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Access token to connect to DOMO - * - * Access Token for Mode Dashboard - */ - accessToken?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * Client ID for DOMO - * - * Azure Application (client) ID for service principal authentication. - * - * User's Client ID. This user should have privileges to read all the metadata in Looker. - * - * client_id for PowerBI. - * - * client_id for Sigma. - * - * Application (client) ID from Azure Active Directory - */ - clientId?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - * - * Secret token to connect to DOMO - */ - secretToken?: string; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * Choose between Database connection or HDB User Store connection. - * - * Choose between API or database connection fetch metadata from superset. - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Matillion Auth Configuration - * - * Choose between mysql and postgres connection for alation database - */ - connection?: ConfigConnection; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * API key to authenticate with the SAP ERP APIs. - * - * API token to connect to Metabase. Use this instead of username/password for token-based - * authentication. - * - * API key of the redash instance to access. - * - * The personal access token you can generate in the Lightdash app under the user settings - * - * Service Account Token to authenticate to the Grafana APIs. Use Service Account Tokens - * (format: glsa_xxxx) for authentication. Legacy API Keys are no longer supported by - * Grafana as of January 2025. Both self-hosted and Grafana Cloud are supported. Requires - * Admin role for full metadata extraction. - * - * Fivetran API Secret. - * - * API Key for Snowplow Console API - */ - apiKey?: string; - /** - * Pagination limit used while querying the SAP ERP API for fetching the entities - * - * Pagination limit used while querying the tableau metadata API for getting data sources - * - * Pagination limit used for Alation APIs pagination - */ - paginationLimit?: number; - /** - * Boolean marking if we need to verify the SSL certs for Grafana. Default to True. - * - * Boolean marking if we need to verify the SSL certs for KafkaConnect REST API. True by - * default. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - */ - verifySSL?: boolean | VerifySSL; - /** - * Azure Application client secret for service principal authentication. - * - * User's Client Secret. - * - * clientSecret for PowerBI. - * - * clientSecret for Sigma. - * - * Application (client) secret from Azure Active Directory - */ - clientSecret?: string; - /** - * Azure Directory (tenant) ID for service principal authentication. - * - * Tenant ID for PowerBI. - * - * Directory (tenant) ID from Azure Active Directory - */ - tenantId?: string; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - */ - gitCredentials?: GitHubCredentials; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * Authority URI for the PowerBI service. - */ - authorityURI?: string; - /** - * Display Table Name from source instead of renamed table name for datamodel tables - */ - displayTableNameFromSource?: boolean; - /** - * Entity Limit set here will be used to paginate the PowerBi APIs - */ - pagination_entity_per_page?: number; - /** - * Source to get the .pbit files to extract lineage information - */ - pbitFilesSource?: PowerBIPbitFilesSource; - /** - * PowerBI secrets. - */ - scope?: string[]; - /** - * Fetch the PowerBI metadata using admin APIs - */ - useAdminApis?: boolean; - /** - * Web Portal Virtual Directory Name. - */ - webPortalVirtualDirectory?: string; - /** - * Version of the Redash instance - */ - redashVersion?: string; - /** - * Tableau API version. If not provided, the version will be used from the tableau server. - * - * Sigma API version. - * - * ThoughtSpot API version to use - * - * Airbyte API version. - * - * OpenMetadata server API version to use. - */ - apiVersion?: string; - /** - * Proxy URL for the tableau server. If not provided, the hostPort will be used. This is - * used to generate the dashboard & Chart URL. - */ - proxyURL?: string; - /** - * Tableau Site Name. - * - * SharePoint site name - */ - siteName?: string; - /** - * Access Token Password for Mode Dashboard - */ - accessTokenPassword?: string; - /** - * Filter query parameter for some of the Mode API calls - */ - filterQueryParam?: string; - /** - * Mode Workspace Name - */ - workspaceName?: string; - /** - * AWS Account ID - */ - awsAccountId?: string; - /** - * The authentication method that the user uses to sign in. - */ - identityType?: IdentityType; - /** - * The Amazon QuickSight namespace that contains the dashboard IDs in this request ( To be - * provided when identityType is `ANONYMOUS` ) - */ - namespace?: string; - certificates?: QlikCertificatesBy; - /** - * Qlik Sense Base URL, used for genrating dashboard & chat url - */ - displayUrl?: string; - /** - * User Directory. - */ - userDirectory?: string; - /** - * User ID. - */ - userId?: string; - /** - * Validate Host Name - */ - validateHostName?: boolean; - /** - * The Project UUID for your Lightdash instance - */ - projectUUID?: string; - /** - * Use if your Lightdash instance is behind a proxy like (Cloud IAP) - */ - proxyAuthentication?: string; - /** - * The Space UUID for your Lightdash instance - */ - spaceUUID?: string; - /** - * Login Mode for Microstrategy's REST API connection. You can authenticate with one of the - * following authentication modes: `Standard (1)`, `Anonymous (8)`. Default will be - * `Standard (1)`. If you're using demo account for Microstrategy, it will be needed to - * authenticate through loginMode `8`. - */ - loginMode?: string; - /** - * MicroStrategy Project Name - * - * Project name to create the refreshToken. Can be anything - */ - projectName?: string; - /** - * Space types of Qlik Cloud to filter the dashboards ingested into the platform. - */ - spaceTypes?: SpaceType[]; - /** - * ThoughtSpot authentication configuration - */ - authentication?: Authenticationation; - /** - * Org ID for multi-tenant ThoughtSpot instances. This is applicable for ThoughtSpot Cloud - * only. - */ - orgId?: string; - /** - * Page size for pagination in API requests. Default is 100. - */ - pageSize?: number; - /** - * basic.auth.user.info schema registry config property, Client HTTP credentials in the form - * of username:password. - */ - basicAuthUserInfo?: string; - /** - * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - * - * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - */ - bootstrapServers?: string; - /** - * Confluent Kafka Consumer Config. From - * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - * - * Confluent Redpanda Consumer Config - */ - consumerConfig?: { [key: string]: any }; - /** - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - */ - consumerConfigSSL?: ConsumerConfigSSLClass; - /** - * sasl.mechanism Consumer Config property - */ - saslMechanism?: SaslMechanismType; - /** - * sasl.password consumer config property - */ - saslPassword?: string; - /** - * sasl.username consumer config property - */ - saslUsername?: string; - /** - * Confluent Kafka Schema Registry Config. From - * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient - * - * Confluent Redpanda Schema Registry Config. - */ - schemaRegistryConfig?: { [key: string]: any }; - /** - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ - schemaRegistrySSL?: ConsumerConfigSSLClass; - /** - * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get - * topic schema from registry. - */ - schemaRegistryTopicSuffixName?: string; - /** - * Confluent Kafka Schema Registry URL. - * - * Confluent Redpanda Schema Registry URL. - */ - schemaRegistryURL?: string; - /** - * security.protocol consumer config property - * - * Kafka security protocol config - */ - securityProtocol?: KafkaSecurityProtocol; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Pipeline Service Number Of Status - */ - numberOfStatus?: number; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Underlying database connection - */ - databaseConnection?: DatabaseConnectionClass; - /** - * Underlying storage connection - */ - packageConnection?: S3Connection | string; - /** - * Fivetran API Secret. - */ - apiSecret?: string; - /** - * Fivetran API Limit For Pagination. - */ - limit?: number; - /** - * URL to the Dagster instance - * - * DBT cloud Access URL. - */ - host?: string; - /** - * Connection Time Limit Between OM and Dagster Graphql API in second - */ - timeout?: number; - /** - * We support username/password or client certificate authentication - */ - nifiConfig?: NifiCredentialsConfiguration; - /** - * Spline UI Host & Port. - */ - uiHostPort?: string; - /** - * service type of the messaging source - */ - brokersUrl?: string; - /** - * consumer group name - */ - consumerGroupName?: string; - /** - * initial Kafka consumer offset - */ - consumerOffsets?: InitialConsumerOffsets; - /** - * max allowed wait time - */ - poolTimeout?: number; - /** - * SASL Configuration details. - */ - saslConfig?: SASLClientConfig; - /** - * max allowed inactivity time - */ - sessionTimeout?: number; - /** - * topic from where Open lineage events will be pulled - */ - topicName?: string; - /** - * We support username/password or No Authentication - */ - KafkaConnectConfig?: UsernamePasswordAuthentication; - /** - * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. - * e.g. local_kafka - * - * service type of the messaging source - */ - messagingServiceName?: string[] | string; - /** - * ID of your DBT cloud account - */ - accountId?: string; - /** - * DBT cloud Metadata API URL. - */ - discoveryAPI?: string; - /** - * List of IDs of your DBT cloud jobs seperated by comma `,` - */ - jobIds?: string[]; - /** - * Number of runs to fetch from DBT cloud - */ - numberOfRuns?: number; - /** - * List of IDs of your DBT cloud projects seperated by comma `,` - */ - projectIds?: string[]; - /** - * The name of your azure data factory. - */ - factory_name?: string; - /** - * The name of your resource group the data factory is associated with. - */ - resource_group_name?: string; - /** - * Number of days in the past to filter pipeline runs. - */ - run_filter_days?: number; - /** - * The azure subscription identifier. - */ - subscription_id?: string; - /** - * Cloud provider where Snowplow is deployed - */ - cloudProvider?: CloudProvider; - /** - * Path to pipeline configuration files for Community deployment - */ - configPath?: string; - /** - * Snowplow Console URL for BDP deployment - */ - consoleUrl?: string; - /** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - */ - deployment?: SnowplowDeployment; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Mlflow Model registry backend. E.g., - * mysql+pymysql://mlflow:password@localhost:3307/experiments - */ - registryUri?: string; - /** - * Mlflow Experiment tracking URI. E.g., http://localhost:5000 - */ - trackingUri?: string; - /** - * location/region of google cloud project - */ - location?: string; - /** - * Enable encryption for the Amundsen Neo4j Connection. - */ - encrypted?: boolean; - /** - * Maximum connection lifetime for the Amundsen Neo4j Connection. - */ - maxConnectionLifeTime?: number; - /** - * Enable SSL validation for the Amundsen Neo4j Connection. - */ - validateSSL?: boolean; - /** - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - /** - * List of entities that you need to reindex - */ - entities?: string[]; - recreateIndex?: boolean; - runMode?: RunMode; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ConfigElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * service type of the data source. - */ - databaseServiceName?: string[]; - /** - * Name of the Entity Type available in Atlas. - */ - entity_type?: string; - /** - * Custom OpenMetadata Classification name for alation tags. - */ - alationTagClassificationName?: string; - /** - * Specifies if hidden datasources should be included while ingesting. - */ - includeHiddenDatasources?: boolean; - /** - * Specifies if undeployed datasources should be included while ingesting. - */ - includeUndeployedDatasources?: boolean; - /** - * Specifies if Dashboards are to be ingested while running the ingestion job. - */ - ingestDashboards?: boolean; - /** - * Specifies if Datasources are to be ingested while running the ingestion job. - */ - ingestDatasources?: boolean; - /** - * Specifies if Domains are to be ingested while running the ingestion job. - */ - ingestDomains?: boolean; - /** - * Specifies if Knowledge Articles are to be ingested while running the ingestion job. - */ - ingestKnowledgeArticles?: boolean; - /** - * Specifies if Users and Groups are to be ingested while running the ingestion job. - */ - ingestUsersAndGroups?: boolean; - datasourceLinks?: { [key: string]: string }; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Connection Timeout in Seconds - */ - connectionTimeoutSecs?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Email to impersonate using domain-wide delegation - */ - delegatedEmail?: string; - /** - * Specific shared drive ID to connect to - * - * SharePoint drive ID. If not provided, default document library will be used - */ - driveId?: string; - /** - * Extract metadata only for Google Sheets files - */ - includeGoogleSheets?: boolean; - /** - * Include shared/team drives in metadata extraction - */ - includeTeamDrives?: boolean; - /** - * SharePoint site URL - */ - siteUrl?: string; - [property: string]: any; -} - -/** - * We support username/password or No Authentication - * - * username/password auth - */ -export interface UsernamePasswordAuthentication { - /** - * KafkaConnect password to authenticate to the API. - */ - password?: string; - /** - * KafkaConnect user to authenticate to the API. - */ - username?: string; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - * - * Types of methods used to authenticate to the tableau instance - * - * Basic Auth Credentials - * - * Access Token Auth Credentials - * - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * API Access Token Auth Credentials - * - * Basic Auth Configuration for ElasticSearch - * - * SSL Certificates By Path - * - * AWS credentials configs. - * - * Authentication type to connect to Apache Ranger. - * - * Configuration for connecting to Ranger Basic Auth. - */ -export interface AuthenticationType { - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - /** - * Password to connect to source. - * - * Password to access the service. - * - * Elastic Search Password for Login - * - * Ranger password to authenticate to the API. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; - /** - * Username to access the service. - * - * Elastic Search Username for Login - * - * Ranger user to authenticate to the API. - */ - username?: string; - /** - * Personal Access Token Name. - */ - personalAccessTokenName?: string; - /** - * Personal Access Token Secret. - */ - personalAccessTokenSecret?: string; - /** - * Access Token for the API - */ - accessToken?: string; - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -/** - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * Basic Auth Credentials - * - * API Access Token Auth Credentials - */ -export interface Authenticationation { - /** - * Password to access the service. - */ - password?: string; - /** - * Username to access the service. - */ - username?: string; - /** - * Access Token for the API - */ - accessToken?: string; -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: Authentication; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum Authentication { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: Credentials | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Qlik Authentication Certificate By Values - * - * Qlik Authentication Certificate File Path - */ -export interface QlikCertificatesBy { - sslConfig?: ConsumerConfigSSLClass; - /** - * Client Certificate - */ - clientCertificate?: string; - /** - * Client Key Certificate. - */ - clientKeyCertificate?: string; - /** - * Root Certificate. - */ - rootCertificate?: string; - [property: string]: any; -} - -/** - * Cloud provider where Snowplow is deployed - */ -export enum CloudProvider { - Aws = "AWS", - Azure = "Azure", - Gcp = "GCP", -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface DeltaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConfigSourceConnection; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: SecurityConfigClass; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConfigSourceConnection { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - * - * GCP Credentials - * - * GCP credentials configs. - * - * GCP Credentials for Google Drive API - * - * AWS credentials configs. - */ -export interface SecurityConfigClass { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - * - * Choose between API or database connection fetch metadata from superset. - * - * Superset API Connection Config - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Lineage Backend Connection Config - * - * SQLite Database Connection Config - * - * Matillion Auth Configuration - * - * Matillion ETL Auth Config. - * - * Choose between mysql and postgres connection for alation database - */ -export interface ConfigConnection { - /** - * Database of the data source. - * - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Host and port of the Hana service. - * - * Host and port of the source service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Matillion Host - */ - hostPort?: string; - /** - * Password to connect to Hana. - * - * Password for Superset. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to the Matillion. - */ - password?: string; - /** - * Username to connect to Hana. This user should have privileges to read all the metadata. - * - * Username for Superset. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to the Matillion. This user should have privileges to read all the - * metadata in Matillion. - */ - username?: string; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; - /** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ - provider?: Provider; - /** - * SSL Configuration details. - */ - sslConfig?: ConnectionSSLConfig; - verifySSL?: VerifySSL; - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: ConnectionScheme; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: ConnectionType; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - supportsViewLineageExtraction?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ -export enum Provider { - DB = "db", - LDAP = "ldap", -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum ConnectionScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", - SqlitePysqlite = "sqlite+pysqlite", -} - -/** - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Configuration for OpenMetadata Server - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ -export interface ConnectionSSLConfig { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum ConnectionType { - Backend = "Backend", - MatillionETL = "MatillionETL", - Mysql = "Mysql", - Postgres = "Postgres", - SQLite = "SQLite", -} - -/** - * initial Kafka consumer offset - */ -export enum InitialConsumerOffsets { - Earliest = "earliest", - Latest = "latest", -} - -/** - * GCP Credentials - * - * GCP credentials configs. - * - * GCP Credentials for Google Drive API - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Underlying database connection - * - * Mssql Database Connection Config - */ -export interface DatabaseConnectionClass { - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Host and port of the MSSQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to MSSQL. - */ - password?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: MssqlScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: MssqlType; - /** - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - */ - username?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum MssqlScheme { - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", -} - -/** - * Service Type - * - * Service type. - */ -export enum MssqlType { - Mssql = "Mssql", -} - -/** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - * - * Snowplow deployment type - */ -export enum SnowplowDeployment { - Bdp = "BDP", - Community = "Community", -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ConfigElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - * - * Do not set any credentials. Note that credentials are required to extract .lkml views and - * their lineage. - * - * Credentials for a GitHub repository - * - * Credentials for a BitBucket repository - * - * Credentials for a Gitlab repository - */ -export interface GitHubCredentials { - repositoryName?: string; - repositoryOwner?: string; - token?: string; - /** - * Credentials Type - */ - type?: GitHubCredentialsType; - /** - * Main production branch of the repository. E.g., `main` - */ - branch?: string; -} - -/** - * Credentials Type - * - * GitHub Credentials type - * - * BitBucket Credentials type - * - * Gitlab Credentials type - */ -export enum GitHubCredentialsType { - BitBucket = "BitBucket", - GitHub = "GitHub", - Gitlab = "Gitlab", -} - -/** - * The authentication method that the user uses to sign in. - */ -export enum IdentityType { - Anonymous = "ANONYMOUS", - Iam = "IAM", - Quicksight = "QUICKSIGHT", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: ConsumerConfigSSLClass; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * We support username/password or client certificate authentication - * - * Configuration for connecting to Nifi Basic Auth. - * - * Configuration for connecting to Nifi Client Certificate Auth. - */ -export interface NifiCredentialsConfiguration { - /** - * Nifi password to authenticate to the API. - */ - password?: string; - /** - * Nifi user to authenticate to the API. - */ - username?: string; - /** - * Boolean marking if we need to verify the SSL certs for Nifi. False by default. - */ - verifySSL?: boolean; - /** - * Path to the root CA certificate - */ - certificateAuthorityPath?: string; - /** - * Path to the client certificate - */ - clientCertificatePath?: string; - /** - * Path to the client key - */ - clientkeyPath?: string; -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * S3 Connection. - */ -export interface S3Connection { - awsConfig: AWSCredentials; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: S3Type; -} - -/** - * Service Type - * - * S3 service type - */ -export enum S3Type { - S3 = "S3", -} - -/** - * Source to get the .pbit files to extract lineage information - * - * Local config source where no extra information needs to be sent. - * - * Azure storage config for pbit files - * - * GCS storage config for pbit files - * - * S3 storage config for pbit files - */ -export interface PowerBIPbitFilesSource { - /** - * Directory path for the pbit files - */ - path?: string; - /** - * pbit File Configuration type - */ - pbitFileConfigType?: PbitFileConfigType; - /** - * Path of the folder where the .pbit files will be unzipped and datamodel schema will be - * extracted - */ - pbitFilesExtractDir?: string; - prefixConfig?: BucketDetails; - securityConfig?: SecurityConfigClass; -} - -/** - * pbit File Configuration type - */ -export enum PbitFileConfigType { - Azure = "azure", - Gcs = "gcs", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the .pbit files are stored - */ -export interface BucketDetails { - /** - * Name of the bucket where the .pbit files are stored - */ - bucketName?: string; - /** - * Path of the folder where the .pbit files are stored - */ - objectPrefix?: string; -} - -/** - * This schema publisher run modes. - */ -export enum RunMode { - Batch = "batch", - Stream = "stream", -} - -/** - * SASL Configuration details. - * - * SASL client configuration. - */ -export interface SASLClientConfig { - /** - * SASL security mechanism - */ - saslMechanism?: SaslMechanismType; - /** - * The SASL authentication password. - */ - saslPassword?: string; - /** - * The SASL authentication username. - */ - saslUsername?: string; -} - -/** - * sasl.mechanism Consumer Config property - * - * SASL Mechanism consumer config property - * - * SASL security mechanism - */ -export enum SaslMechanismType { - Gssapi = "GSSAPI", - Oauthbearer = "OAUTHBEARER", - Plain = "PLAIN", - ScramSHA256 = "SCRAM-SHA-256", - ScramSHA512 = "SCRAM-SHA-512", -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * security.protocol consumer config property - * - * Kafka security protocol config - */ -export enum KafkaSecurityProtocol { - Plaintext = "PLAINTEXT", - SSL = "SSL", - SaslPlaintext = "SASL_PLAINTEXT", - SaslSSL = "SASL_SSL", -} - -export enum SpaceType { - Data = "Data", - Managed = "Managed", - Personal = "Personal", - Shared = "Shared", -} - -/** - * SSL Configuration for OpenMetadata Server - * - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Config - */ -export interface SSLConfigObject { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; - /** - * SSL Certificates - */ - certificates?: SSLCertificates; - [property: string]: any; -} - -/** - * SSL Certificates - * - * SSL Configuration details. - * - * SSL Certificates By Path - * - * SSL Certificates By Values - */ -export interface SSLCertificates { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * CA Certificate Value - */ - caCertValue?: string; - /** - * Client Certificate Value - */ - clientCertValue?: string; - /** - * Private Key Value - */ - privateKeyValue?: string; - /** - * Staging Directory Path - */ - stagingDir?: string; -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * REST API Type - * - * REST API type - * - * Service Type - * - * Service type. - * - * service type - * - * Custom database service type - * - * Looker service type - * - * Metabase service type - * - * PowerBI service type - * - * PowerBIReportServer service type - * - * Redash service type - * - * Superset service type - * - * Tableau service type - * - * Mode service type - * - * Custom dashboard service type - * - * QuickSight service type - * - * Qlik sense service type - * - * Lightdash service type - * - * MicroStrategy service type - * - * Qlik Cloud service type - * - * Sigma service type - * - * ThoughtSpot service type - * - * Grafana service type - * - * Kafka service type - * - * Redpanda service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Amundsen service type - * - * Metadata to Elastic Search type - * - * OpenMetadata service type - * - * S3 service type - * - * ADLS service type - * - * Gcs service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * ElasticSearch service type - * - * OpenSearch Type - * - * OpenSearch service type - * - * Custom search service type - * - * Apache Ranger service type - * - * Google Drive service type - * - * SharePoint service type - * - * Custom Drive service type - */ -export enum ConfigType { - Adls = "ADLS", - Airbyte = "Airbyte", - Airflow = "Airflow", - Alation = "Alation", - AlationSink = "AlationSink", - Amundsen = "Amundsen", - Athena = "Athena", - Atlas = "Atlas", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDashboard = "CustomDashboard", - CustomDatabase = "CustomDatabase", - CustomDrive = "CustomDrive", - CustomMessaging = "CustomMessaging", - CustomMlModel = "CustomMlModel", - CustomPipeline = "CustomPipeline", - CustomSearch = "CustomSearch", - CustomStorage = "CustomStorage", - DBTCloud = "DBTCloud", - Dagster = "Dagster", - DataFactory = "DataFactory", - Databricks = "Databricks", - DatabricksPipeline = "DatabricksPipeline", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDashboard = "DomoDashboard", - DomoDatabase = "DomoDatabase", - DomoPipeline = "DomoPipeline", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - ElasticSearch = "ElasticSearch", - Epic = "Epic", - Exasol = "Exasol", - Fivetran = "Fivetran", - Flink = "Flink", - Gcs = "GCS", - Glue = "Glue", - GluePipeline = "GluePipeline", - GoogleDrive = "GoogleDrive", - Grafana = "Grafana", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - Kafka = "Kafka", - KafkaConnect = "KafkaConnect", - Kinesis = "Kinesis", - Lightdash = "Lightdash", - Looker = "Looker", - MariaDB = "MariaDB", - Matillion = "Matillion", - Metabase = "Metabase", - MetadataES = "MetadataES", - MicroStrategy = "MicroStrategy", - Mlflow = "Mlflow", - Mode = "Mode", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Nifi = "Nifi", - OpenLineage = "OpenLineage", - OpenMetadata = "OpenMetadata", - OpenSearch = "OpenSearch", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - PowerBI = "PowerBI", - PowerBIReportServer = "PowerBIReportServer", - Presto = "Presto", - QlikCloud = "QlikCloud", - QlikSense = "QlikSense", - QuickSight = "QuickSight", - REST = "Rest", - Ranger = "Ranger", - Redash = "Redash", - Redpanda = "Redpanda", - Redshift = "Redshift", - S3 = "S3", - SAS = "SAS", - SQLite = "SQLite", - SageMaker = "SageMaker", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SharePoint = "SharePoint", - Sigma = "Sigma", - SingleStore = "SingleStore", - Sklearn = "Sklearn", - Snowflake = "Snowflake", - Snowplow = "Snowplow", - Spark = "Spark", - Spline = "Spline", - Ssas = "SSAS", - Ssis = "SSIS", - Stitch = "Stitch", - Superset = "Superset", - Synapse = "Synapse", - Tableau = "Tableau", - Teradata = "Teradata", - ThoughtSpot = "ThoughtSpot", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - VertexAI = "VertexAI", - Vertica = "Vertica", - Wherescape = "Wherescape", -} - -/** - * Operation to be performed on the entity - */ -export interface Operation { - /** - * Entity to be modified - */ - entityLink: string; - /** - * The id of the operation - */ - id: string; - /** - * The configuration for the operation to be applied - */ - parameters: ReverseIngestionConfig; - /** - * Templated SQL command to be used for the operation. Context parameters will be populated - * based on the event type. - */ - SQLTemplate?: string; - /** - * Type of operation to perform - */ - type: OperationType; -} - -/** - * The configuration for the operation to be applied - * - * Configuration for updating descriptions - * - * Configuration for updating owners - * - * Configuration for updating tags - */ -export interface ReverseIngestionConfig { - /** - * New description of the service - */ - newDescription?: string; - /** - * Previous description of the service - */ - previousDescription?: string; - /** - * Added owners to be applied - */ - addedOwners?: EntityReference[]; - /** - * Removed owners from the entity - */ - removedOwners?: EntityReference[]; - /** - * Added tags to be applied - */ - addedTags?: TagLabel[]; - /** - * Removed tags of the entity - */ - removedTags?: TagLabel[]; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - */ -export interface TagLabel { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelType; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelType { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * Type of operation to perform - */ -export enum OperationType { - UpdateDescription = "UPDATE_DESCRIPTION", - UpdateOwner = "UPDATE_OWNER", - UpdateTags = "UPDATE_TAGS", -} - -/** - * Type of service such as Database, Dashboard, Messaging, etc. - * - * This schema defines the service types entities which requires a connection. - */ -export enum ServiceType { - API = "Api", - Dashboard = "Dashboard", - Database = "Database", - Drive = "Drive", - Messaging = "Messaging", - Metadata = "Metadata", - MlModel = "MlModel", - Pipeline = "Pipeline", - Search = "Search", - Security = "Security", - Storage = "Storage", -} - -/** - * Spark Engine Configuration. - * - * This schema defines the configuration for a Spark Engine runner. - */ -export interface SparkEngineConfiguration { - config?: Config; - /** - * Spark Connect Remote URL. - */ - remote: string; - type: SparkEngineType; -} - -export interface Config { - /** - * Additional Spark configuration properties as key-value pairs. - */ - extraConfig?: { [key: string]: any }; - /** - * Temporary path to store the data. - */ - tempPath?: string; - [property: string]: any; -} - -export enum SparkEngineType { - Spark = "Spark", -} - -/** - * Pipeline type - * - * Reverse Ingestion Config Pipeline type - */ -export enum ReverseIngestionType { - ReverseIngestion = "ReverseIngestion", -} - -/** - * Response to the request. - * - * TestConnectionResult is the definition that will encapsulate result of running the test - * connection steps. - * - * Apply a set of operations on a service - * - * Query Runner Response - */ -export interface TestConnectionResult { - /** - * Last time that the test connection was executed - */ - lastUpdatedAt?: number; - /** - * Test Connection Result computation status. - * - * Status of the query execution - */ - status?: StatusType; - /** - * Steps to test the connection. Order matters. - */ - steps?: TestConnectionStepResult[]; - /** - * Error message in case of failure - */ - message?: string; - /** - * List of operations to be performed on the service - * - * Results of the query execution - */ - results?: ReverseIngestionOperationResult[] | TableData; - /** - * The id of the service to be modified - */ - serviceId?: string; - /** - * Whether the workflow was successful. Failure indicates a critical failure such as - * connection issues. - */ - success?: boolean; - /** - * Duration of the query execution in seconds - */ - duration?: number; - /** - * Detailed error log in case of failure - */ - errorLog?: string; -} - -export interface ReverseIngestionOperationResult { - /** - * The id of the operation - */ - id: string; - /** - * Error message in case of failure - */ - message?: string; - /** - * Whether the specific operation was successful - */ - success: boolean; - [property: string]: any; -} - -/** - * Results of the query execution - * - * This schema defines the type to capture rows of sample data for a table. - */ -export interface TableData { - /** - * List of local column names (not fully qualified column names) of the table. - */ - columns?: string[]; - /** - * Data for multiple rows of the table. - */ - rows?: Array; -} - -/** - * Test Connection Result computation status. - * - * Enum defining possible Test Connection Result status - * - * Status of the query execution - * - * Enum defining possible Query Runner status - */ -export enum StatusType { - Failed = "Failed", - Running = "Running", - Successful = "Successful", -} - -/** - * Function that tests one specific element of the service. E.g., listing schemas, lineage, - * or tags. - */ -export interface TestConnectionStepResult { - /** - * In case of failed step, this field would contain the actual error faced during the step. - */ - errorLog?: string; - /** - * Is this step mandatory to be passed? - */ - mandatory: boolean; - /** - * Results or exceptions to be shared after running the test. This message comes from the - * test connection definition - */ - message?: string; - /** - * Name of the step being tested - */ - name: string; - /** - * Did the step pass successfully? - */ - passed: boolean; -} - -/** - * Workflow computation status. - * - * Enum defining possible Workflow status - */ -export enum WorkflowStatus { - Failed = "Failed", - Pending = "Pending", - Running = "Running", - Successful = "Successful", -} - -/** - * Type of the workflow. - * - * This enum defines the type for which this workflow applies to. - */ -export enum WorkflowType { - QueryRunner = "QUERY_RUNNER", - ReverseIngestion = "REVERSE_INGESTION", - TestConnection = "TEST_CONNECTION", - TestSparkEngineConnection = "TEST_SPARK_ENGINE_CONNECTION", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/spreadsheet.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/spreadsheet.ts index e6270e5e4e6..ef3310a5e05 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/spreadsheet.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/spreadsheet.ts @@ -409,6 +409,7 @@ export enum EntityStatus { Draft = "Draft", InReview = "In Review", Rejected = "Rejected", + Unprocessed = "Unprocessed", } /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/unityCatalogConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/unityCatalogConnection.ts deleted file mode 100644 index 95ce89897fc..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/unityCatalogConnection.ts +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * UnityCatalog Connection Config - */ -export interface UnityCatalogConnection { - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - */ - catalog?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - */ - connectionTimeout?: number; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Host and port of the Databricks service. - */ - hostPort: string; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: DatabricksScheme; - supportsDatabase?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Generated Token to connect to Databricks. - */ - token: string; - /** - * Service Type - */ - type?: DatabricksType; -} - -/** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum DatabricksScheme { - DatabricksConnector = "databricks+connector", -} - -/** - * Service Type - * - * Service type. - */ -export enum DatabricksType { - UnityCatalog = "UnityCatalog", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts deleted file mode 100644 index 221f82490ec..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts +++ /dev/null @@ -1,4115 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Groups source connection configurations. - */ -export interface ServiceConnection { - /** - * Service Connection. - */ - serviceConnection?: ServiceConnectionClass; - [property: string]: any; -} - -/** - * Service Connection. - * - * Supported services - * - * API Service Connection. - * - * Dashboard Connection. - * - * Database Connection. - * - * Metadata Service Connection. - * - * Pipeline Connection. - * - * MlModel Connection. - * - * Storage Connection. - * - * search Connection. - * - * Security Connection. - * - * Drive Connection. - */ -export interface ServiceConnectionClass { - config?: ConfigObject; -} - -/** - * REST Connection Config - * - * Looker Connection Config - * - * Metabase Connection Config - * - * PowerBI Connection Config - * - * PowerBIReportServer Connection Config - * - * Redash Connection Config - * - * Superset Connection Config - * - * Tableau Connection Config - * - * Mode Connection Config - * - * Custom Dashboard Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Domo Dashboard Connection Config - * - * QuickSight Connection Config - * - * Qlik Sense Connection Config - * - * Lightdash Connection Config - * - * MicroStrategy Connection Config - * - * Qlik Cloud Connection Config - * - * Sigma Connection Config - * - * ThoughtSpot Connection Config - * - * Grafana Connection Config - * - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - * - * Kafka Connection Config - * - * Redpanda Connection Config - * - * Kinesis Connection Config - * - * Custom Messaging Service Connection to build a source that is not supported by - * OpenMetadata yet. - * - * Amundsen Connection Config - * - * Metadata to ElasticSearch Connection Config - * - * OpenMetadata Connection Config - * - * Atlas Connection Config - * - * Alation Connection Config - * - * Alation Sink Connection Config - * - * Airflow Metadata Database Connection Config - * - * Wherescape Metadata Database Connection Config - * - * SSIS Metadata Database Connection Config - * - * Glue Pipeline Connection Config - * - * Airbyte Metadata Database Connection Config - * - * Fivetran Metadata Database Connection Config - * - * Flink Metadata Connection Config - * - * Dagster Metadata Database Connection Config - * - * Nifi Metadata Pipeline Connection Config - * - * Domo Pipeline Connection Config - * - * Custom Pipeline Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Spline Metadata Database Connection Config - * - * Spark Metadata Pipeline Connection Config - * - * OpenLineage Connection Config - * - * KafkaConnect Connection Config - * - * DBTCloud Connection Config - * - * Matillion Connection - * - * Azure Data Factory Connection Config - * - * Stitch Connection - * - * Snowplow Pipeline Connection Config - * - * MlFlow Connection Config - * - * Sklearn Connection Config - * - * Custom MlModel Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * SageMaker Connection Config - * - * Google VertexAI Connection Config - * - * S3 Connection. - * - * ADLS Connection. - * - * GCS Connection. - * - * Custom Storage Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * ElasticSearch Connection. - * - * OpenSearch Connection Config - * - * Custom Search Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * Apache Ranger Connection Config - * - * Google Drive Connection Config - * - * SharePoint Connection Config - * - * Custom Drive Connection to build a source that is not supported. - */ -export interface ConfigObject { - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Documentation URL for the schema. - */ - docURL?: string; - /** - * Open API Schema URL. - */ - openAPISchemaURL?: string; - /** - * Supports Metadata Extraction. - */ - supportsMetadataExtraction?: boolean; - /** - * Generated Token to connect to OpenAPI Schema. - * - * token to connect to Qlik Cloud. - * - * Generated Token to connect to Databricks. - * - * To Connect to Dagster Cloud - * - * Generated Token to connect to DBTCloud. - * - * Token to connect to Stitch api doc - */ - token?: string; - /** - * REST API Type - * - * Service Type - * - * Custom dashboard service type - * - * Custom database service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * OpenSearch Type - * - * Custom search service type - */ - type?: ConfigType; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * User's Client ID. This user should have privileges to read all the metadata in Looker. - * - * client_id for PowerBI. - * - * Client ID for DOMO - * - * client_id for Sigma. - * - * Azure Application (client) ID for service principal authentication. - * - * Application (client) ID from Azure Active Directory - */ - clientId?: string; - /** - * User's Client Secret. - * - * clientSecret for PowerBI. - * - * clientSecret for Sigma. - * - * Azure Application client secret for service principal authentication. - * - * Application (client) secret from Azure Active Directory - */ - clientSecret?: string; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - */ - gitCredentials?: GitHubCredentials; - /** - * URL to the Looker instance. - * - * Host and Port of the Metabase instance. - * - * Dashboard URL for PowerBI service. - * - * Dashboard URL for PowerBI Report Server. - * - * URL for the Redash instance - * - * URL for the superset instance. - * - * Tableau Server url. - * - * URL for the mode instance. - * - * URL for the Qlik instance. - * - * Address for your running Lightdash instance - * - * Host and Port of the MicroStrategy instance. - * - * Host and Port of the Qlik Cloud instance. - * - * Sigma API url. - * - * ThoughtSpot instance URL. Example: https://my-company.thoughtspot.cloud - * - * URL to the Grafana instance. - * - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - * - * Host and port of the Amundsen Neo4j Connection. This expect a URI format like: - * bolt://localhost:7687. - * - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - * - * Host and port of the Atlas service. - * - * Host and port of the Alation service. - * - * Pipeline Service Management/UI URI. - * - * Pipeline Service Management/UI URL. - * - * Spline REST Server Host & Port. - * - * KafkaConnect Service Management/UI URI. - * - * Host and port of the Stitch API host - * - * Host and port of the ElasticSearch service. - * - * Host and port of the OpenSearch service. - * - * Apache Ranger Admin URL. - */ - hostPort?: string; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * API token to connect to Metabase. Use this instead of username/password for token-based - * authentication. - * - * API key of the redash instance to access. - * - * The personal access token you can generate in the Lightdash app under the user settings - * - * Service Account Token to authenticate to the Grafana APIs. Use Service Account Tokens - * (format: glsa_xxxx) for authentication. Legacy API Keys are no longer supported by - * Grafana as of January 2025. Both self-hosted and Grafana Cloud are supported. Requires - * Admin role for full metadata extraction. - * - * API key to authenticate with the SAP ERP APIs. - * - * Fivetran API Secret. - * - * API Key for Snowplow Console API - */ - apiKey?: string; - /** - * Password to connect to Metabase. Required for basic authentication. - * - * Password to connect to PowerBI report server. - * - * Password to connect to MicroStrategy. - * - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to the Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - * - * password to connect to the Amundsen Neo4j Connection. - * - * password to connect to the Atlas. - * - * Password to connect to Airbyte. - */ - password?: string; - /** - * Username to connect to Metabase. Required for basic authentication. - * - * Username to connect to PowerBI report server. - * - * Username for Redash - * - * Username to connect to MicroStrategy. This user should have privileges to read all the - * metadata in MicroStrategy. - * - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to the Salesforce. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - * - * username to connect to the Amundsen Neo4j Connection. - * - * username to connect to the Atlas. This user should have privileges to read all the - * metadata in Atlas. - * - * Username to connect to Airbyte. - */ - username?: string; - /** - * Authority URI for the PowerBI service. - */ - authorityURI?: string; - /** - * Display Table Name from source instead of renamed table name for datamodel tables - */ - displayTableNameFromSource?: boolean; - /** - * Entity Limit set here will be used to paginate the PowerBi APIs - */ - pagination_entity_per_page?: number; - /** - * Source to get the .pbit files to extract lineage information - */ - pbitFilesSource?: PowerBIPbitFilesSource; - /** - * PowerBI secrets. - */ - scope?: string[]; - /** - * Tenant ID for PowerBI. - * - * Azure Directory (tenant) ID for service principal authentication. - * - * Directory (tenant) ID from Azure Active Directory - */ - tenantId?: string; - /** - * Fetch the PowerBI metadata using admin APIs - */ - useAdminApis?: boolean; - /** - * Web Portal Virtual Directory Name. - */ - webPortalVirtualDirectory?: string; - /** - * Version of the Redash instance - */ - redashVersion?: string; - /** - * Choose between API or database connection fetch metadata from superset. - * - * Choose between Database connection or HDB User Store connection. - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Matillion Auth Configuration - */ - connection?: ConfigConnection; - /** - * Tableau API version. If not provided, the version will be used from the tableau server. - * - * Sigma API version. - * - * ThoughtSpot API version to use - * - * OpenMetadata server API version to use. - * - * Airbyte API version. - */ - apiVersion?: string; - /** - * Types of methods used to authenticate to the tableau instance - * - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - * - * Types of methods used to authenticate to the alation instance - * - * Authentication type to connect to Apache Ranger. - */ - authType?: AuthenticationTypeForTableau | NoConfigAuthenticationTypes; - /** - * Pagination limit used while querying the tableau metadata API for getting data sources - * - * Pagination limit used while querying the SAP ERP API for fetching the entities - * - * Pagination limit used for Alation APIs pagination - */ - paginationLimit?: number; - /** - * Proxy URL for the tableau server. If not provided, the hostPort will be used. This is - * used to generate the dashboard & Chart URL. - */ - proxyURL?: string; - /** - * Tableau Site Name. - * - * SharePoint site name - */ - siteName?: string; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; - /** - * Boolean marking if we need to verify the SSL certs for Grafana. Default to True. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Boolean marking if we need to verify the SSL certs for KafkaConnect REST API. True by - * default. - */ - verifySSL?: boolean | VerifySSL; - /** - * Access Token for Mode Dashboard - * - * Access token to connect to DOMO - */ - accessToken?: string; - /** - * Access Token Password for Mode Dashboard - */ - accessTokenPassword?: string; - /** - * Filter query parameter for some of the Mode API calls - */ - filterQueryParam?: string; - /** - * Mode Workspace Name - */ - workspaceName?: string; - connectionOptions?: { [key: string]: string }; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - * - * Secret token to connect to DOMO - */ - secretToken?: string; - /** - * AWS Account ID - */ - awsAccountId?: string; - awsConfig?: AWSCredentials; - /** - * The authentication method that the user uses to sign in. - */ - identityType?: IdentityType; - /** - * The Amazon QuickSight namespace that contains the dashboard IDs in this request ( To be - * provided when identityType is `ANONYMOUS` ) - */ - namespace?: string; - certificates?: QlikCertificatesBy; - /** - * Qlik Sense Base URL, used for genrating dashboard & chat url - */ - displayUrl?: string; - /** - * User Directory. - */ - userDirectory?: string; - /** - * User ID. - */ - userId?: string; - /** - * Validate Host Name - */ - validateHostName?: boolean; - /** - * The Project UUID for your Lightdash instance - */ - projectUUID?: string; - /** - * Use if your Lightdash instance is behind a proxy like (Cloud IAP) - */ - proxyAuthentication?: string; - /** - * The Space UUID for your Lightdash instance - */ - spaceUUID?: string; - /** - * Login Mode for Microstrategy's REST API connection. You can authenticate with one of the - * following authentication modes: `Standard (1)`, `Anonymous (8)`. Default will be - * `Standard (1)`. If you're using demo account for Microstrategy, it will be needed to - * authenticate through loginMode `8`. - */ - loginMode?: string; - /** - * MicroStrategy Project Name - * - * Project name to create the refreshToken. Can be anything - */ - projectName?: string; - /** - * Space types of Qlik Cloud to filter the dashboards ingested into the platform. - */ - spaceTypes?: SpaceType[]; - /** - * ThoughtSpot authentication configuration - */ - authentication?: Authenticationation; - /** - * Org ID for multi-tenant ThoughtSpot instances. This is applicable for ThoughtSpot Cloud - * only. - */ - orgId?: string; - /** - * Page size for pagination in API requests. Default is 100. - */ - pageSize?: number; - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - * - * Azure Credentials - * - * GCP Credentials for Google Drive API - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - * - * Connection timeout in seconds. - */ - connectionTimeout?: number | number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - * - * Available sources to fetch metadata. - */ - configSource?: DeltaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - */ - auth?: AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - * - * Snowplow BDP Organization ID - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - /** - * basic.auth.user.info schema registry config property, Client HTTP credentials in the form - * of username:password. - */ - basicAuthUserInfo?: string; - /** - * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - * - * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - */ - bootstrapServers?: string; - /** - * Confluent Kafka Consumer Config. From - * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - * - * Confluent Redpanda Consumer Config - */ - consumerConfig?: { [key: string]: any }; - /** - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - */ - consumerConfigSSL?: ConsumerConfigSSLClass; - /** - * sasl.mechanism Consumer Config property - */ - saslMechanism?: SaslMechanismType; - /** - * sasl.password consumer config property - */ - saslPassword?: string; - /** - * sasl.username consumer config property - */ - saslUsername?: string; - /** - * Confluent Kafka Schema Registry Config. From - * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient - * - * Confluent Redpanda Schema Registry Config. - */ - schemaRegistryConfig?: { [key: string]: any }; - /** - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ - schemaRegistrySSL?: ConsumerConfigSSLClass; - /** - * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get - * topic schema from registry. - */ - schemaRegistryTopicSuffixName?: string; - /** - * Confluent Kafka Schema Registry URL. - * - * Confluent Redpanda Schema Registry URL. - */ - schemaRegistryURL?: string; - /** - * security.protocol consumer config property - * - * Kafka security protocol config - */ - securityProtocol?: KafkaSecurityProtocol; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Enable encryption for the Amundsen Neo4j Connection. - */ - encrypted?: boolean; - /** - * Maximum connection lifetime for the Amundsen Neo4j Connection. - */ - maxConnectionLifeTime?: number; - /** - * Enable SSL validation for the Amundsen Neo4j Connection. - */ - validateSSL?: boolean; - /** - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - /** - * List of entities that you need to reindex - */ - entities?: string[]; - recreateIndex?: boolean; - runMode?: RunMode; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * service type of the data source. - */ - databaseServiceName?: string[]; - /** - * Name of the Entity Type available in Atlas. - */ - entity_type?: string; - /** - * service type of the messaging source - * - * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. - * e.g. local_kafka - */ - messagingServiceName?: string[] | string; - /** - * Custom OpenMetadata Classification name for alation tags. - */ - alationTagClassificationName?: string; - /** - * Specifies if hidden datasources should be included while ingesting. - */ - includeHiddenDatasources?: boolean; - /** - * Specifies if undeployed datasources should be included while ingesting. - */ - includeUndeployedDatasources?: boolean; - /** - * Specifies if Dashboards are to be ingested while running the ingestion job. - */ - ingestDashboards?: boolean; - /** - * Specifies if Datasources are to be ingested while running the ingestion job. - */ - ingestDatasources?: boolean; - /** - * Specifies if Domains are to be ingested while running the ingestion job. - */ - ingestDomains?: boolean; - /** - * Specifies if Knowledge Articles are to be ingested while running the ingestion job. - */ - ingestKnowledgeArticles?: boolean; - /** - * Specifies if Users and Groups are to be ingested while running the ingestion job. - */ - ingestUsersAndGroups?: boolean; - datasourceLinks?: { [key: string]: string }; - /** - * Pipeline Service Number Of Status - */ - numberOfStatus?: number; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Underlying database connection - */ - databaseConnection?: DatabaseConnectionClass; - /** - * Underlying storage connection - */ - packageConnection?: S3Connection | string; - /** - * Fivetran API Secret. - */ - apiSecret?: string; - /** - * Fivetran API Limit For Pagination. - */ - limit?: number; - /** - * URL to the Dagster instance - * - * DBT cloud Access URL. - */ - host?: string; - /** - * Connection Time Limit Between OM and Dagster Graphql API in second - */ - timeout?: number; - /** - * We support username/password or client certificate authentication - */ - nifiConfig?: NifiCredentialsConfiguration; - /** - * Spline UI Host & Port. - */ - uiHostPort?: string; - /** - * service type of the messaging source - */ - brokersUrl?: string; - /** - * consumer group name - */ - consumerGroupName?: string; - /** - * initial Kafka consumer offset - */ - consumerOffsets?: InitialConsumerOffsets; - /** - * max allowed wait time - */ - poolTimeout?: number; - /** - * SASL Configuration details. - */ - saslConfig?: SASLClientConfig; - /** - * max allowed inactivity time - */ - sessionTimeout?: number; - /** - * topic from where Open lineage events will be pulled - */ - topicName?: string; - /** - * We support username/password or No Authentication - */ - KafkaConnectConfig?: UsernamePasswordAuthentication; - /** - * ID of your DBT cloud account - */ - accountId?: string; - /** - * DBT cloud Metadata API URL. - */ - discoveryAPI?: string; - /** - * List of IDs of your DBT cloud jobs seperated by comma `,` - */ - jobIds?: string[]; - /** - * Number of runs to fetch from DBT cloud - */ - numberOfRuns?: number; - /** - * List of IDs of your DBT cloud projects seperated by comma `,` - */ - projectIds?: string[]; - /** - * The name of your azure data factory. - */ - factory_name?: string; - /** - * The name of your resource group the data factory is associated with. - */ - resource_group_name?: string; - /** - * Number of days in the past to filter pipeline runs. - */ - run_filter_days?: number; - /** - * The azure subscription identifier. - */ - subscription_id?: string; - /** - * Cloud provider where Snowplow is deployed - */ - cloudProvider?: CloudProvider; - /** - * Path to pipeline configuration files for Community deployment - */ - configPath?: string; - /** - * Snowplow Console URL for BDP deployment - */ - consoleUrl?: string; - /** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - */ - deployment?: SnowplowDeployment; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Mlflow Model registry backend. E.g., - * mysql+pymysql://mlflow:password@localhost:3307/experiments - */ - registryUri?: string; - /** - * Mlflow Experiment tracking URI. E.g., http://localhost:5000 - */ - trackingUri?: string; - /** - * location/region of google cloud project - */ - location?: string; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Connection Timeout in Seconds - */ - connectionTimeoutSecs?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Email to impersonate using domain-wide delegation - */ - delegatedEmail?: string; - /** - * Specific shared drive ID to connect to - * - * SharePoint drive ID. If not provided, default document library will be used - */ - driveId?: string; - /** - * Extract metadata only for Google Sheets files - */ - includeGoogleSheets?: boolean; - /** - * Include shared/team drives in metadata extraction - */ - includeTeamDrives?: boolean; - /** - * SharePoint site URL - */ - siteUrl?: string; - [property: string]: any; -} - -/** - * We support username/password or No Authentication - * - * username/password auth - */ -export interface UsernamePasswordAuthentication { - /** - * KafkaConnect password to authenticate to the API. - */ - password?: string; - /** - * KafkaConnect user to authenticate to the API. - */ - username?: string; -} - -/** - * Regex to only fetch api collections with names matching the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex exclude or include charts that matches the pattern. - * - * Regex to exclude or include dashboards that matches the pattern. - * - * Regex exclude or include data models that matches the pattern. - * - * Regex to exclude or include projects that matches the pattern. - * - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - * - * Regex to only fetch topics that matches the pattern. - * - * Regex exclude pipelines. - * - * Regex to only fetch containers that matches the pattern. - * - * Regex to only fetch MlModels with names matching the pattern. - * - * Regex to only fetch search indexes that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Types of methods used to authenticate to the tableau instance - * - * Basic Auth Credentials - * - * Access Token Auth Credentials - * - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - * - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * API Access Token Auth Credentials - * - * Basic Auth Configuration for ElasticSearch - * - * SSL Certificates By Path - * - * AWS credentials configs. - * - * Authentication type to connect to Apache Ranger. - * - * Configuration for connecting to Ranger Basic Auth. - */ -export interface AuthenticationTypeForTableau { - /** - * Password to access the service. - * - * Password to connect to source. - * - * Elastic Search Password for Login - * - * Ranger password to authenticate to the API. - */ - password?: string; - /** - * Username to access the service. - * - * Elastic Search Username for Login - * - * Ranger user to authenticate to the API. - */ - username?: string; - /** - * Personal Access Token Name. - */ - personalAccessTokenName?: string; - /** - * Personal Access Token Secret. - */ - personalAccessTokenSecret?: string; - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; - /** - * Access Token for the API - */ - accessToken?: string; - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -/** - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * Basic Auth Credentials - * - * API Access Token Auth Credentials - */ -export interface Authenticationation { - /** - * Password to access the service. - */ - password?: string; - /** - * Username to access the service. - */ - username?: string; - /** - * Access Token for the API - */ - accessToken?: string; -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: Authentication; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum Authentication { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: Credentials | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Qlik Authentication Certificate By Values - * - * Qlik Authentication Certificate File Path - */ -export interface QlikCertificatesBy { - sslConfig?: ConsumerConfigSSLClass; - /** - * Client Certificate - */ - clientCertificate?: string; - /** - * Client Key Certificate. - */ - clientKeyCertificate?: string; - /** - * Root Certificate. - */ - rootCertificate?: string; - [property: string]: any; -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface ConsumerConfigSSLClass { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Cloud provider where Snowplow is deployed - */ -export enum CloudProvider { - Aws = "AWS", - Azure = "Azure", - Gcp = "GCP", -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface DeltaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConfigSourceConnection; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: SecurityConfigClass; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConfigSourceConnection { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - * - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * AWS credentials configs. - */ -export interface SecurityConfigClass { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Choose between API or database connection fetch metadata from superset. - * - * Superset API Connection Config - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - * - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Lineage Backend Connection Config - * - * SQLite Database Connection Config - * - * Matillion Auth Configuration - * - * Matillion ETL Auth Config. - */ -export interface ConfigConnection { - /** - * Password for Superset. - * - * Password to connect to Hana. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to the Matillion. - */ - password?: string; - /** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ - provider?: Provider; - /** - * SSL Configuration details. - */ - sslConfig?: ConnectionSSLConfig; - /** - * Username for Superset. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to Hana. This user should have privileges to read all the metadata. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to the Matillion. This user should have privileges to read all the - * metadata in Matillion. - */ - username?: string; - verifySSL?: VerifySSL; - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - * - * Host and port of the Hana service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Matillion Host - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: ConnectionScheme; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: ConnectionType; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - supportsViewLineageExtraction?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ -export enum Provider { - DB = "db", - LDAP = "ldap", -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum ConnectionScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", - SqlitePysqlite = "sqlite+pysqlite", -} - -/** - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - */ -export interface ConnectionSSLConfig { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum ConnectionType { - Backend = "Backend", - MatillionETL = "MatillionETL", - Mysql = "Mysql", - Postgres = "Postgres", - SQLite = "SQLite", -} - -/** - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * initial Kafka consumer offset - */ -export enum InitialConsumerOffsets { - Earliest = "earliest", - Latest = "latest", -} - -/** - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Underlying database connection - * - * Mssql Database Connection Config - */ -export interface DatabaseConnectionClass { - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Host and port of the MSSQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to MSSQL. - */ - password?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: MssqlScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: MssqlType; - /** - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - */ - username?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum MssqlScheme { - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", -} - -/** - * Service Type - * - * Service type. - */ -export enum MssqlType { - Mssql = "Mssql", -} - -/** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - * - * Snowplow deployment type - */ -export enum SnowplowDeployment { - Bdp = "BDP", - Community = "Community", -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - * - * Do not set any credentials. Note that credentials are required to extract .lkml views and - * their lineage. - * - * Credentials for a GitHub repository - * - * Credentials for a BitBucket repository - * - * Credentials for a Gitlab repository - */ -export interface GitHubCredentials { - repositoryName?: string; - repositoryOwner?: string; - token?: string; - /** - * Credentials Type - */ - type?: GitHubCredentialsType; - /** - * Main production branch of the repository. E.g., `main` - */ - branch?: string; -} - -/** - * Credentials Type - * - * GitHub Credentials type - * - * BitBucket Credentials type - * - * Gitlab Credentials type - */ -export enum GitHubCredentialsType { - BitBucket = "BitBucket", - GitHub = "GitHub", - Gitlab = "Gitlab", -} - -/** - * The authentication method that the user uses to sign in. - */ -export enum IdentityType { - Anonymous = "ANONYMOUS", - Iam = "IAM", - Quicksight = "QUICKSIGHT", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: ConsumerConfigSSLClass; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * We support username/password or client certificate authentication - * - * Configuration for connecting to Nifi Basic Auth. - * - * Configuration for connecting to Nifi Client Certificate Auth. - */ -export interface NifiCredentialsConfiguration { - /** - * Nifi password to authenticate to the API. - */ - password?: string; - /** - * Nifi user to authenticate to the API. - */ - username?: string; - /** - * Boolean marking if we need to verify the SSL certs for Nifi. False by default. - */ - verifySSL?: boolean; - /** - * Path to the root CA certificate - */ - certificateAuthorityPath?: string; - /** - * Path to the client certificate - */ - clientCertificatePath?: string; - /** - * Path to the client key - */ - clientkeyPath?: string; -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * S3 Connection. - */ -export interface S3Connection { - awsConfig: AWSCredentials; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: S3Type; -} - -/** - * Service Type - * - * S3 service type - */ -export enum S3Type { - S3 = "S3", -} - -/** - * Source to get the .pbit files to extract lineage information - * - * Local config source where no extra information needs to be sent. - * - * Azure storage config for pbit files - * - * GCS storage config for pbit files - * - * S3 storage config for pbit files - */ -export interface PowerBIPbitFilesSource { - /** - * Directory path for the pbit files - */ - path?: string; - /** - * pbit File Configuration type - */ - pbitFileConfigType?: PbitFileConfigType; - /** - * Path of the folder where the .pbit files will be unzipped and datamodel schema will be - * extracted - */ - pbitFilesExtractDir?: string; - prefixConfig?: BucketDetails; - securityConfig?: SecurityConfigClass; -} - -/** - * pbit File Configuration type - */ -export enum PbitFileConfigType { - Azure = "azure", - Gcs = "gcs", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the .pbit files are stored - */ -export interface BucketDetails { - /** - * Name of the bucket where the .pbit files are stored - */ - bucketName?: string; - /** - * Path of the folder where the .pbit files are stored - */ - objectPrefix?: string; -} - -/** - * This schema publisher run modes. - */ -export enum RunMode { - Batch = "batch", - Stream = "stream", -} - -/** - * SASL Configuration details. - * - * SASL client configuration. - */ -export interface SASLClientConfig { - /** - * SASL security mechanism - */ - saslMechanism?: SaslMechanismType; - /** - * The SASL authentication password. - */ - saslPassword?: string; - /** - * The SASL authentication username. - */ - saslUsername?: string; -} - -/** - * sasl.mechanism Consumer Config property - * - * SASL Mechanism consumer config property - * - * SASL security mechanism - */ -export enum SaslMechanismType { - Gssapi = "GSSAPI", - Oauthbearer = "OAUTHBEARER", - Plain = "PLAIN", - ScramSHA256 = "SCRAM-SHA-256", - ScramSHA512 = "SCRAM-SHA-512", -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * security.protocol consumer config property - * - * Kafka security protocol config - */ -export enum KafkaSecurityProtocol { - Plaintext = "PLAINTEXT", - SSL = "SSL", - SaslPlaintext = "SASL_PLAINTEXT", - SaslSSL = "SASL_SSL", -} - -export enum SpaceType { - Data = "Data", - Managed = "Managed", - Personal = "Personal", - Shared = "Shared", -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Config - */ -export interface SSLConfigObject { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; - /** - * SSL Certificates - */ - certificates?: SSLCertificates; - [property: string]: any; -} - -/** - * SSL Certificates - * - * SSL Configuration details. - * - * SSL Certificates By Path - * - * SSL Certificates By Values - */ -export interface SSLCertificates { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * CA Certificate Value - */ - caCertValue?: string; - /** - * Client Certificate Value - */ - clientCertValue?: string; - /** - * Private Key Value - */ - privateKeyValue?: string; - /** - * Staging Directory Path - */ - stagingDir?: string; -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * REST API Type - * - * REST API type - * - * Service Type - * - * Looker service type - * - * Metabase service type - * - * PowerBI service type - * - * PowerBIReportServer service type - * - * Redash service type - * - * Superset service type - * - * Tableau service type - * - * Mode service type - * - * Custom dashboard service type - * - * service type - * - * QuickSight service type - * - * Qlik sense service type - * - * Lightdash service type - * - * MicroStrategy service type - * - * Qlik Cloud service type - * - * Sigma service type - * - * ThoughtSpot service type - * - * Grafana service type - * - * Service type. - * - * Custom database service type - * - * Kafka service type - * - * Redpanda service type - * - * Custom messaging service type - * - * Amundsen service type - * - * Metadata to Elastic Search type - * - * OpenMetadata service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * S3 service type - * - * ADLS service type - * - * Gcs service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * ElasticSearch service type - * - * OpenSearch Type - * - * OpenSearch service type - * - * Custom search service type - * - * Apache Ranger service type - * - * Google Drive service type - * - * SharePoint service type - * - * Custom Drive service type - */ -export enum ConfigType { - Adls = "ADLS", - Airbyte = "Airbyte", - Airflow = "Airflow", - Alation = "Alation", - AlationSink = "AlationSink", - Amundsen = "Amundsen", - Athena = "Athena", - Atlas = "Atlas", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDashboard = "CustomDashboard", - CustomDatabase = "CustomDatabase", - CustomDrive = "CustomDrive", - CustomMessaging = "CustomMessaging", - CustomMlModel = "CustomMlModel", - CustomPipeline = "CustomPipeline", - CustomSearch = "CustomSearch", - CustomStorage = "CustomStorage", - DBTCloud = "DBTCloud", - Dagster = "Dagster", - DataFactory = "DataFactory", - Databricks = "Databricks", - DatabricksPipeline = "DatabricksPipeline", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDashboard = "DomoDashboard", - DomoDatabase = "DomoDatabase", - DomoPipeline = "DomoPipeline", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - ElasticSearch = "ElasticSearch", - Epic = "Epic", - Exasol = "Exasol", - Fivetran = "Fivetran", - Flink = "Flink", - Gcs = "GCS", - Glue = "Glue", - GluePipeline = "GluePipeline", - GoogleDrive = "GoogleDrive", - Grafana = "Grafana", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - Kafka = "Kafka", - KafkaConnect = "KafkaConnect", - Kinesis = "Kinesis", - Lightdash = "Lightdash", - Looker = "Looker", - MariaDB = "MariaDB", - Matillion = "Matillion", - Metabase = "Metabase", - MetadataES = "MetadataES", - MicroStrategy = "MicroStrategy", - Mlflow = "Mlflow", - Mode = "Mode", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Nifi = "Nifi", - OpenLineage = "OpenLineage", - OpenMetadata = "OpenMetadata", - OpenSearch = "OpenSearch", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - PowerBI = "PowerBI", - PowerBIReportServer = "PowerBIReportServer", - Presto = "Presto", - QlikCloud = "QlikCloud", - QlikSense = "QlikSense", - QuickSight = "QuickSight", - REST = "Rest", - Ranger = "Ranger", - Redash = "Redash", - Redpanda = "Redpanda", - Redshift = "Redshift", - S3 = "S3", - SAS = "SAS", - SQLite = "SQLite", - SageMaker = "SageMaker", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SharePoint = "SharePoint", - Sigma = "Sigma", - SingleStore = "SingleStore", - Sklearn = "Sklearn", - Snowflake = "Snowflake", - Snowplow = "Snowplow", - Spark = "Spark", - Spline = "Spline", - Ssas = "SSAS", - Ssis = "SSIS", - Stitch = "Stitch", - Superset = "Superset", - Synapse = "Synapse", - Tableau = "Tableau", - Teradata = "Teradata", - ThoughtSpot = "ThoughtSpot", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - VertexAI = "VertexAI", - Vertica = "Vertica", - Wherescape = "Wherescape", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts deleted file mode 100644 index 9a24548a64c..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts +++ /dev/null @@ -1,2463 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This schema defines the `Database Service` is a service such as MySQL, BigQuery, - * Redshift, Postgres, or Snowflake. Alternative terms such as Database Cluster, Database - * Server instance are also used for database service. - */ -export interface DatabaseService { - /** - * Change that lead to this version of the entity. - */ - changeDescription?: ChangeDescription; - connection?: DatabaseConnection; - /** - * List of data products this entity is part of. - */ - dataProducts?: EntityReference[]; - /** - * When `true` indicates the entity has been soft deleted. - */ - deleted?: boolean; - /** - * Description of a database service instance. - */ - description?: string; - /** - * Display Name that identifies this database service. - */ - displayName?: string; - /** - * Domains the Database service belongs to. - */ - domains?: EntityReference[]; - /** - * Followers of this entity. - */ - followers?: EntityReference[]; - /** - * FullyQualifiedName same as `name`. - */ - fullyQualifiedName?: string; - /** - * Link to the resource corresponding to this database service. - */ - href?: string; - /** - * Unique identifier of this database service instance. - */ - id: string; - /** - * Change that lead to this version of the entity. - */ - incrementalChangeDescription?: ChangeDescription; - /** - * The ingestion agent responsible for executing the ingestion pipeline. - */ - ingestionRunner?: EntityReference; - /** - * Name that identifies this database service. - */ - name: string; - /** - * Owners of this database service. - */ - owners?: EntityReference[]; - /** - * References to pipelines deployed for this database service to extract metadata, usage, - * lineage etc.. - */ - pipelines?: EntityReference[]; - /** - * Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres... - */ - serviceType: DatabaseServiceType; - /** - * Tags for this Database Service. - */ - tags?: TagLabel[]; - /** - * Last test connection results for this service - */ - testConnectionResult?: TestConnectionResult; - /** - * Last update time corresponding to the new version of the entity in Unix epoch time - * milliseconds. - */ - updatedAt?: number; - /** - * User who made the update. - */ - updatedBy?: string; - /** - * Metadata version of the entity. - */ - version?: number; -} - -/** - * Change that lead to this version of the entity. - * - * Description of the change. - */ -export interface ChangeDescription { - changeSummary?: { [key: string]: ChangeSummary }; - /** - * Names of fields added during the version changes. - */ - fieldsAdded?: FieldChange[]; - /** - * Fields deleted during the version changes with old value before deleted. - */ - fieldsDeleted?: FieldChange[]; - /** - * Fields modified during the version changes with old and new values. - */ - fieldsUpdated?: FieldChange[]; - /** - * When a change did not result in change, this could be same as the current version. - */ - previousVersion?: number; -} - -export interface ChangeSummary { - changedAt?: number; - /** - * Name of the user or bot who made this change - */ - changedBy?: string; - changeSource?: ChangeSource; - [property: string]: any; -} - -/** - * The source of the change. This will change based on the context of the change (example: - * manual vs programmatic) - */ -export enum ChangeSource { - Automated = "Automated", - Derived = "Derived", - Ingested = "Ingested", - Manual = "Manual", - Propagated = "Propagated", - Suggested = "Suggested", -} - -export interface FieldChange { - /** - * Name of the entity field that changed. - */ - name?: string; - /** - * New value of the field. Note that this is a JSON string and use the corresponding field - * type to deserialize it. - */ - newValue?: any; - /** - * Previous value of the field. Note that this is a JSON string and use the corresponding - * field type to deserialize it. - */ - oldValue?: any; -} - -/** - * Database Connection. - */ -export interface DatabaseConnection { - config?: ConfigObject; -} - -/** - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - */ -export interface ConfigObject { - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - */ - hostPort?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Service Type - * - * Custom database service type - */ - type?: ConfigType; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - awsConfig?: AWSCredentials; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to the Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - */ - password?: string; - /** - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to the Salesforce. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - */ - username?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - */ - authType?: AuthenticationType | NoConfigAuthenticationTypes; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - */ - connectionTimeout?: number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - */ - configSource?: TaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - */ - auth?: AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * SSL Configuration details. - */ - sslConfig?: Config; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Access token to connect to DOMO - */ - accessToken?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * Client ID for DOMO - * - * Azure Application (client) ID for service principal authentication. - */ - clientId?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - */ - secretToken?: string; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * Choose between Database connection or HDB User Store connection. - */ - connection?: SAPHanaConnection; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Generated Token to connect to Databricks. - */ - token?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * API key to authenticate with the SAP ERP APIs. - */ - apiKey?: string; - /** - * Pagination limit used while querying the SAP ERP API for fetching the entities - */ - paginationLimit?: number; - verifySSL?: VerifySSL; - /** - * Azure Application client secret for service principal authentication. - */ - clientSecret?: string; - /** - * Azure Directory (tenant) ID for service principal authentication. - */ - tenantId?: string; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - [property: string]: any; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface AuthenticationType { - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: Authentication; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum Authentication { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: Credentials | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface TaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConnectionClass; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: SecurityConfigClass; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConnectionClass { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Azure Cloud Credentials - * - * GCP Credentials - * - * GCP credentials configs. - * - * AWS credentials configs. - */ -export interface SecurityConfigClass { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - */ -export interface SAPHanaConnection { - /** - * Database of the data source. - */ - database?: string; - /** - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Host and port of the Hana service. - */ - hostPort?: string; - /** - * Password to connect to Hana. - */ - password?: string; - /** - * Username to connect to Hana. This user should have privileges to read all the metadata. - */ - username?: string; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; -} - -/** - * GCP Credentials - * - * GCP credentials configs. - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; -} - -/** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: Config; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface Config { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * Service Type - * - * Service type. - * - * service type - * - * Custom database service type - */ -export enum ConfigType { - Athena = "Athena", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDatabase = "CustomDatabase", - Databricks = "Databricks", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDatabase = "DomoDatabase", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - Epic = "Epic", - Exasol = "Exasol", - Glue = "Glue", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - MariaDB = "MariaDB", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - Presto = "Presto", - Redshift = "Redshift", - SAS = "SAS", - SQLite = "SQLite", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SingleStore = "SingleStore", - Snowflake = "Snowflake", - Ssas = "SSAS", - Synapse = "Synapse", - Teradata = "Teradata", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - Vertica = "Vertica", -} - -/** - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * List of data products this entity is part of. - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * The ingestion agent responsible for executing the ingestion pipeline. - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres... - */ -export enum DatabaseServiceType { - Athena = "Athena", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDatabase = "CustomDatabase", - Databricks = "Databricks", - Datalake = "Datalake", - Db2 = "Db2", - Dbt = "Dbt", - DeltaLake = "DeltaLake", - DomoDatabase = "DomoDatabase", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - Epic = "Epic", - Exasol = "Exasol", - Glue = "Glue", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - MariaDB = "MariaDB", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - Presto = "Presto", - QueryLog = "QueryLog", - Redshift = "Redshift", - SAS = "SAS", - SQLite = "SQLite", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SingleStore = "SingleStore", - Snowflake = "Snowflake", - Ssas = "SSAS", - Synapse = "Synapse", - Teradata = "Teradata", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - Vertica = "Vertica", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - */ -export interface TagLabel { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelType; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelType { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * Last test connection results for this service - * - * TestConnectionResult is the definition that will encapsulate result of running the test - * connection steps. - */ -export interface TestConnectionResult { - /** - * Last time that the test connection was executed - */ - lastUpdatedAt?: number; - /** - * Test Connection Result computation status. - */ - status?: StatusType; - /** - * Steps to test the connection. Order matters. - */ - steps: TestConnectionStepResult[]; -} - -/** - * Test Connection Result computation status. - * - * Enum defining possible Test Connection Result status - */ -export enum StatusType { - Failed = "Failed", - Running = "Running", - Successful = "Successful", -} - -/** - * Function that tests one specific element of the service. E.g., listing schemas, lineage, - * or tags. - */ -export interface TestConnectionStepResult { - /** - * In case of failed step, this field would contain the actual error faced during the step. - */ - errorLog?: string; - /** - * Is this step mandatory to be passed? - */ - mandatory: boolean; - /** - * Results or exceptions to be shared after running the test. This message comes from the - * test connection definition - */ - message?: string; - /** - * Name of the step being tested - */ - name: string; - /** - * Did the step pass successfully? - */ - passed: boolean; -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts deleted file mode 100644 index 5225c8255d8..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts +++ /dev/null @@ -1,6568 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Ingestion Pipeline Config is used to set up a DAG and deploy. This entity is used to - * setup metadata/quality pipelines on Apache Airflow. - */ -export interface IngestionPipeline { - airflowConfig: AirflowConfig; - /** - * Type of the application when pipelineType is 'application'. - */ - applicationType?: string; - /** - * Change that led to this version of the entity. - */ - changeDescription?: ChangeDescription; - /** - * When `true` indicates the entity has been soft deleted. - */ - deleted?: boolean; - /** - * Indicates if the workflow has been successfully deployed to Airflow. - */ - deployed?: boolean; - /** - * Description of the Pipeline. - */ - description?: string; - /** - * Display Name that identifies this Pipeline. - */ - displayName?: string; - /** - * Domains the asset belongs to. When not set, the asset inherits the domain from the parent - * it belongs to. - */ - domains?: EntityReference[]; - /** - * True if the pipeline is ready to be run in the next schedule. False if it is paused. - */ - enabled?: boolean; - /** - * Enable real-time log streaming to the OpenMetadata server. When enabled, ingestion logs - * will be automatically shipped to the server's configured log storage backend (S3 or - * compatible). - */ - enableStreamableLogs?: boolean; - /** - * Followers of this entity. - */ - followers?: EntityReference[]; - /** - * Name that uniquely identifies a Pipeline. - */ - fullyQualifiedName?: string; - /** - * Link to this ingestion pipeline resource. - */ - href?: string; - /** - * Unique identifier that identifies this pipeline. - */ - id?: string; - /** - * Change that lead to this version of the entity. - */ - incrementalChangeDescription?: ChangeDescription; - /** - * The ingestion agent responsible for executing the ingestion pipeline. - */ - ingestionRunner?: EntityReference; - /** - * Set the logging level for the workflow. - */ - loggerLevel?: LogLevels; - /** - * Name that identifies this pipeline instance uniquely. - */ - name: string; - openMetadataServerConnection?: OpenMetadataConnection; - /** - * Owners of this Pipeline. - */ - owners?: EntityReference[]; - /** - * Last of executions and status for the Pipeline. - */ - pipelineStatuses?: PipelineStatus; - pipelineType: PipelineType; - /** - * The processing engine responsible for executing the ingestion pipeline logic. - */ - processingEngine?: EntityReference; - provider?: ProviderType; - /** - * Control if we want to flag the workflow as failed if we encounter any processing errors. - */ - raiseOnError?: boolean; - /** - * Link to the service (such as database, messaging, storage services, etc. for which this - * ingestion pipeline ingests the metadata from. - */ - service?: EntityReference; - sourceConfig: SourceConfig; - /** - * Last update time corresponding to the new version of the entity in Unix epoch time - * milliseconds. - */ - updatedAt?: number; - /** - * User who made the update. - */ - updatedBy?: string; - /** - * Metadata version of the entity. - */ - version?: number; -} - -/** - * Properties to configure the Airflow pipeline that will run the workflow. - */ -export interface AirflowConfig { - /** - * Concurrency of the Pipeline. - */ - concurrency?: number; - /** - * Email to notify workflow status. - */ - email?: string; - /** - * End Date of the pipeline. - */ - endDate?: Date; - /** - * Maximum Number of active runs. - */ - maxActiveRuns?: number; - /** - * pause the pipeline from running once the deploy is finished successfully. - */ - pausePipeline?: boolean; - /** - * Run past executions if the start date is in the past. - */ - pipelineCatchup?: boolean; - /** - * Timezone in which pipeline going to be scheduled. - */ - pipelineTimezone?: string; - /** - * Retry pipeline in case of failure. - */ - retries?: number; - /** - * Delay between retries in seconds. - */ - retryDelay?: number; - /** - * Scheduler Interval for the pipeline in cron format. - */ - scheduleInterval?: string; - /** - * Start date of the pipeline. - */ - startDate?: Date; - /** - * Default view in Airflow. - */ - workflowDefaultView?: string; - /** - * Default view Orientation in Airflow. - */ - workflowDefaultViewOrientation?: string; - /** - * Timeout for the workflow in seconds. - */ - workflowTimeout?: number; -} - -/** - * Change that led to this version of the entity. - * - * Description of the change. - * - * Change that lead to this version of the entity. - */ -export interface ChangeDescription { - changeSummary?: { [key: string]: ChangeSummary }; - /** - * Names of fields added during the version changes. - */ - fieldsAdded?: FieldChange[]; - /** - * Fields deleted during the version changes with old value before deleted. - */ - fieldsDeleted?: FieldChange[]; - /** - * Fields modified during the version changes with old and new values. - */ - fieldsUpdated?: FieldChange[]; - /** - * When a change did not result in change, this could be same as the current version. - */ - previousVersion?: number; -} - -export interface ChangeSummary { - changedAt?: number; - /** - * Name of the user or bot who made this change - */ - changedBy?: string; - changeSource?: ChangeSource; - [property: string]: any; -} - -/** - * The source of the change. This will change based on the context of the change (example: - * manual vs programmatic) - */ -export enum ChangeSource { - Automated = "Automated", - Derived = "Derived", - Ingested = "Ingested", - Manual = "Manual", - Propagated = "Propagated", - Suggested = "Suggested", -} - -export interface FieldChange { - /** - * Name of the entity field that changed. - */ - name?: string; - /** - * New value of the field. Note that this is a JSON string and use the corresponding field - * type to deserialize it. - */ - newValue?: any; - /** - * Previous value of the field. Note that this is a JSON string and use the corresponding - * field type to deserialize it. - */ - oldValue?: any; -} - -/** - * Domains the asset belongs to. When not set, the asset inherits the domain from the parent - * it belongs to. - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * The ingestion agent responsible for executing the ingestion pipeline. - * - * The processing engine responsible for executing the ingestion pipeline logic. - * - * Link to the service (such as database, messaging, storage services, etc. for which this - * ingestion pipeline ingests the metadata from. - * - * Service to be modified - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Set the logging level for the workflow. - * - * Supported logging levels - */ -export enum LogLevels { - Debug = "DEBUG", - Error = "ERROR", - Info = "INFO", - Warn = "WARN", -} - -/** - * OpenMetadata Connection Config - */ -export interface OpenMetadataConnection { - /** - * OpenMetadata server API version to use. - */ - apiVersion?: string; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: OpenMetadataServerConnectionElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - */ - hostPort: string; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: ConsumerConfigSSLClass; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: OpenmetadataType; - /** - * Flag to verify SSL Certificate for OpenMetadata Server. - */ - verifySSL?: VerifySSL; -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to only fetch databases that matches the pattern. - * - * Regex to only fetch tables or databases that matches the pattern. - * - * Regex to only fetch stored procedures that matches the pattern. - * - * Regex exclude tables or databases that matches the pattern. - * - * Regex exclude or include charts that matches the pattern. - * - * Regex to exclude or include dashboards that matches the pattern. - * - * Regex exclude or include data models that matches the pattern. - * - * Regex to exclude or include projects that matches the pattern. - * - * Regex to only fetch topics that matches the pattern. - * - * Regex to only compute metrics for table that matches the given tag, tiers, gloassary - * pattern. - * - * Regex exclude pipelines. - * - * Regex to only fetch MlModels with names matching the pattern. - * - * Regex to only fetch containers that matches the pattern. - * - * Regex to only include/exclude directories that matches the pattern. - * - * Regex to only include/exclude files that matches the pattern. - * - * Regex to only include/exclude spreadsheets that matches the pattern. - * - * Regex to only include/exclude worksheets that matches the pattern. - * - * Regex to only fetch search indexes that matches the pattern. - * - * Regex to only fetch api collections with names matching the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - * - * Regex to only fetch tags that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface OpenMetadataServerConnectionElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * SSL Configuration for OpenMetadata Server - * - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface ConsumerConfigSSLClass { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Service Type - * - * OpenMetadata service type - */ -export enum OpenmetadataType { - OpenMetadata = "OpenMetadata", -} - -/** - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * Last of executions and status for the Pipeline. - * - * This defines runtime status of Pipeline. - */ -export interface PipelineStatus { - /** - * Pipeline configuration for this particular execution. - */ - config?: { [key: string]: any }; - /** - * endDate of the pipeline run for this particular execution. - */ - endDate?: number; - /** - * Metadata for the pipeline status. - */ - metadata?: { [key: string]: any }; - /** - * Pipeline status denotes if its failed or succeeded. - */ - pipelineState?: PipelineState; - /** - * Pipeline unique run ID. - */ - runId?: string; - /** - * startDate of the pipeline run for this particular execution. - */ - startDate?: number; - /** - * Ingestion Pipeline summary status. Informed at the end of the execution. - */ - status?: StepSummary[]; - /** - * executionDate of the pipeline run for this particular execution. - */ - timestamp?: number; -} - -/** - * Pipeline status denotes if its failed or succeeded. - */ -export enum PipelineState { - Failed = "failed", - PartialSuccess = "partialSuccess", - Queued = "queued", - Running = "running", - Success = "success", -} - -/** - * Ingestion Pipeline summary status. Informed at the end of the execution. - * - * Summary for each step of the ingestion pipeline - * - * Defines the summary status of each step executed in an Ingestion Pipeline. - */ -export interface StepSummary { - /** - * Number of records with errors. - */ - errors?: number; - /** - * Sample of errors encountered in the step - */ - failures?: StackTraceError[]; - /** - * Number of filtered records. - */ - filtered?: number; - /** - * Step name - */ - name: string; - /** - * Number of successfully processed records. - */ - records?: number; - /** - * Number of successfully updated records. - */ - updated_records?: number; - /** - * Number of records raising warnings. - */ - warnings?: number; -} - -/** - * Represents a failure status - */ -export interface StackTraceError { - /** - * Error being handled - */ - error: string; - /** - * Name of the asset with the error - */ - name: string; - /** - * Exception stack trace - */ - stackTrace?: string; -} - -/** - * Type of Pipeline - metadata, usage - */ -export enum PipelineType { - Application = "application", - AutoClassification = "autoClassification", - DataInsight = "dataInsight", - Dbt = "dbt", - ElasticSearchReindex = "elasticSearchReindex", - Lineage = "lineage", - Metadata = "metadata", - Profiler = "profiler", - TestSuite = "TestSuite", - Usage = "usage", -} - -/** - * Type of provider of an entity. Some entities are provided by the `system`. Some are - * entities created and provided by the `user`. Typically `system` provide entities can't be - * deleted and can only be disabled. Some apps such as AutoPilot create entities with - * `automation` provider type. These entities can be deleted by the user. - */ -export enum ProviderType { - Automation = "automation", - System = "system", - User = "user", -} - -/** - * Additional connection configuration. - */ -export interface SourceConfig { - config?: Pipeline; -} - -/** - * DatabaseService Metadata Pipeline Configuration. - * - * DatabaseService Query Usage Pipeline Configuration. - * - * DatabaseService Query Lineage Pipeline Configuration. - * - * DashboardService Metadata Pipeline Configuration. - * - * MessagingService Metadata Pipeline Configuration. - * - * DatabaseService Profiler Pipeline Configuration. - * - * DatabaseService AutoClassification & Auto Classification Pipeline Configuration. - * - * PipelineService Metadata Pipeline Configuration. - * - * MlModelService Metadata Pipeline Configuration. - * - * StorageService Metadata Pipeline Configuration. - * - * DriveService Metadata Pipeline Configuration. - * - * SearchService Metadata Pipeline Configuration. - * - * TestSuite Pipeline Configuration. - * - * Data Insight Pipeline Configuration. - * - * DBT Pipeline Configuration. - * - * Application Pipeline Configuration. - * - * ApiService Metadata Pipeline Configuration. - * - * Apply a set of operations on a service - */ -export interface Pipeline { - /** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Optional configuration to toggle the DDL Statements ingestion. - */ - includeDDL?: boolean; - /** - * Set the 'Include Owners' toggle to control whether to include owners to the ingested - * entity if the owner email matches with a user stored in the OM server as part of metadata - * ingestion. If the ingested entity already exists and has an owner, the owner will not be - * overwritten. - * - * Enabling a flag will replace the current owner with a new owner from the source during - * metadata ingestion, if the current owner is null. It is recommended to keep the flag - * enabled to obtain the owner information during the first metadata ingestion. - */ - includeOwners?: boolean; - /** - * Optional configuration to toggle the Stored Procedures ingestion. - */ - includeStoredProcedures?: boolean; - /** - * Optional configuration to turn off fetching metadata for tables. - */ - includeTables?: boolean; - /** - * Optional configuration to toggle the tags ingestion. - */ - includeTags?: boolean; - /** - * Optional configuration to turn off fetching metadata for views. - */ - includeViews?: boolean; - /** - * Use incremental Metadata extraction after the first execution. This is commonly done by - * getting the changes from Audit tables on the supporting databases. - */ - incremental?: IncrementalMetadataExtractionConfiguration; - /** - * Optional configuration to soft delete databases in OpenMetadata if the source databases - * are deleted. Also, if the database is deleted, all the associated entities like schemas, - * tables, views, stored procedures, lineage, etc., with that database will be deleted - */ - markDeletedDatabases?: boolean; - /** - * Optional configuration to soft delete schemas in OpenMetadata if the source schemas are - * deleted. Also, if the schema is deleted, all the associated entities like tables, views, - * stored procedures, lineage, etc., with that schema will be deleted - */ - markDeletedSchemas?: boolean; - /** - * Optional configuration to soft delete stored procedures in OpenMetadata if the source - * stored procedures are deleted. Also, if the stored procedures is deleted, all the - * associated entities like lineage, etc., with that stored procedures will be deleted - */ - markDeletedStoredProcedures?: boolean; - /** - * This is an optional configuration for enabling soft deletion of tables. When this option - * is enabled, only tables that have been deleted from the source will be soft deleted, and - * this will apply solely to the schema that is currently being ingested via the pipeline. - * Any related entities such as test suites or lineage information that were associated with - * those tables will also be deleted. - */ - markDeletedTables?: boolean; - /** - * Set the 'Override Metadata' toggle to control whether to override the existing metadata - * in the OpenMetadata server with the metadata fetched from the source. If the toggle is - * set to true, the metadata fetched from the source will override the existing metadata in - * the OpenMetadata server. If the toggle is set to false, the metadata fetched from the - * source will not override the existing metadata in the OpenMetadata server. This is - * applicable for fields like description, tags, owner and displayName - */ - overrideMetadata?: boolean; - /** - * Configuration to tune how far we want to look back in query logs to process Stored - * Procedures results. - * - * Configuration to tune how far we want to look back in query logs to process usage data. - * - * Configuration to tune how far we want to look back in query logs to process lineage data. - */ - queryLogDuration?: number; - /** - * Configuration to set the timeout for parsing the query in seconds. - */ - queryParsingTimeoutLimit?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only fetch tables or databases that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex exclude tables or databases that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Number of Threads to use in order to parallelize Table ingestion. - * - * Number of Threads to use in order to parallelize lineage ingestion. - * - * Number of Threads to use in order to parallelize Drive ingestion. - */ - threads?: number; - /** - * Pipeline type - */ - type?: FluffyType; - /** - * Regex will be applied on fully qualified name (e.g - * service_name.db_name.schema_name.table_name) instead of raw name (e.g. table_name) - * - * Regex will be applied on fully qualified name (e.g service_name.directory_name.file_name) - * instead of raw name (e.g. file_name) - */ - useFqnForFiltering?: boolean; - /** - * Configuration the condition to filter the query history. - */ - filterCondition?: string; - /** - * Configuration to process query cost - */ - processQueryCostAnalysis?: boolean; - /** - * Configuration to set the file path for query logs - */ - queryLogFilePath?: string; - /** - * Configuration to set the limit for query logs - */ - resultLimit?: number; - /** - * Temporary file name to store the query logs before processing. Absolute file path - * required. - */ - stageFileLocation?: string; - /** - * Set 'Cross Database Service Names' to process lineage with the database. - */ - crossDatabaseServiceNames?: string[]; - /** - * Handle Lineage for Snowflake Temporary and Transient Tables. - */ - enableTempTableLineage?: boolean; - /** - * Set the 'Incremental Lineage Processing' toggle to control whether to process lineage - * incrementally. - */ - incrementalLineageProcessing?: boolean; - /** - * Set the 'Override View Lineage' toggle to control whether to override the existing view - * lineage. - */ - overrideViewLineage?: boolean; - /** - * Configuration to set the timeout for parsing the query in seconds. - */ - parsingTimeoutLimit?: number; - /** - * Set the 'Process Cross Database Lineage' toggle to control whether to process table - * lineage across different databases. - */ - processCrossDatabaseLineage?: boolean; - /** - * Set the 'Process Query Lineage' toggle to control whether to process query lineage. - */ - processQueryLineage?: boolean; - /** - * Set the 'Process Stored ProcedureLog Lineage' toggle to control whether to process stored - * procedure lineage. - */ - processStoredProcedureLineage?: boolean; - /** - * Set the 'Process View Lineage' toggle to control whether to process view lineage. - */ - processViewLineage?: boolean; - /** - * Regex to only fetch stored procedures that matches the pattern. - */ - storedProcedureFilterPattern?: FilterPattern; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Optional configuration to toggle the ingestion of data models. - */ - includeDataModels?: boolean; - /** - * Optional Configuration to include/exclude draft dashboards. By default it will include - * draft dashboards - */ - includeDraftDashboard?: boolean; - /** - * Details required to generate Lineage - */ - lineageInformation?: LineageInformation; - /** - * Optional configuration to soft delete dashboards in OpenMetadata if the source dashboards - * are deleted. Also, if the dashboard is deleted, all the associated entities like lineage, - * etc., with that dashboard will be deleted - */ - markDeletedDashboards?: boolean; - /** - * Optional configuration to soft delete data models in OpenMetadata if the source data - * models are deleted. Also, if the data models is deleted, all the associated entities like - * lineage, etc., with that data models will be deleted - */ - markDeletedDataModels?: boolean; - /** - * Set the 'Override Lineage' toggle to control whether to override the existing lineage. - */ - overrideLineage?: boolean; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * Option to turn on/off generating sample data during metadata extraction. - */ - generateSampleData?: boolean; - /** - * Optional configuration to soft delete topics in OpenMetadata if the source topics are - * deleted. Also, if the topic is deleted, all the associated entities like sample data, - * lineage, etc., with that topic will be deleted - */ - markDeletedTopics?: boolean; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Regex to only compute metrics for table that matches the given tag, tiers, gloassary - * pattern. - */ - classificationFilterPattern?: FilterPattern; - /** - * Option to turn on/off column metric computation. If enabled, profiler will compute column - * level metrics. - */ - computeColumnMetrics?: boolean; - /** - * Option to turn on/off computing profiler metrics. - */ - computeMetrics?: boolean; - /** - * Option to turn on/off table metric computation. If enabled, profiler will compute table - * level metrics. - */ - computeTableMetrics?: boolean; - processingEngine?: ProcessingEngine; - /** - * Percentage of data or no. of rows used to compute the profiler metrics and run data - * quality tests - * - * Percentage of data or no. of rows we want to execute the profiler and tests on - */ - profileSample?: number; - profileSampleType?: ProfileSampleType; - /** - * Whether to randomize the sample data or not. - */ - randomizedSample?: boolean; - samplingMethodType?: SamplingMethodType; - /** - * Number of threads to use during metric computations - */ - threadCount?: number; - /** - * Profiler Timeout in Seconds - */ - timeoutSeconds?: number; - /** - * Use system tables to extract metrics. Metrics that cannot be gathered from system tables - * will use the default methods. Using system tables can be faster but requires gathering - * statistics before running (for example using the ANALYZE procedure). More information can - * be found in the documentation: https://docs.openmetadata.org/latest/profler - */ - useStatistics?: boolean; - /** - * Set the Confidence value for which you want the column to be tagged as PII. Confidence - * value ranges from 0 to 100. A higher number will yield less false positives but more - * false negatives. A lower number will yield more false positives but less false negatives. - */ - confidence?: number; - /** - * Optional configuration to automatically tag columns that might contain sensitive - * information - */ - enableAutoClassification?: boolean; - /** - * Number of sample rows to ingest when 'Generate Sample Data' is enabled - */ - sampleDataCount?: number; - /** - * Option to turn on/off storing sample data. If enabled, we will ingest sample data for - * each table. - */ - storeSampleData?: boolean; - /** - * Optional configuration to turn off fetching lineage from pipelines. - */ - includeLineage?: boolean; - /** - * Optional configuration to toggle whether the un-deployed pipelines should be ingested or - * not. If set to false, only deployed pipelines will be ingested. - */ - includeUnDeployedPipelines?: boolean; - /** - * Optional configuration to soft delete Pipelines in OpenMetadata if the source Pipelines - * are deleted. Also, if the Pipeline is deleted, all the associated entities like lineage, - * etc., with that Pipeline will be deleted - */ - markDeletedPipelines?: boolean; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete MlModels in OpenMetadata if the source MlModels are - * deleted. Also, if the MlModel is deleted, all the associated entities like lineage, etc., - * with that MlModels will be deleted - */ - markDeletedMlModels?: boolean; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete containers in OpenMetadata if the source containers - * are deleted. Also, if the topic is deleted, all the associated entities with that - * containers will be deleted - */ - markDeletedContainers?: boolean; - storageMetadataConfigSource?: StorageMetadataConfigurationSource; - /** - * Regex to only include/exclude directories that matches the pattern. - */ - directoryFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude files that matches the pattern. - */ - fileFilterPattern?: FilterPattern; - /** - * Optional configuration to turn off fetching metadata for directories. - */ - includeDirectories?: boolean; - /** - * Optional configuration to turn off fetching metadata for files. - */ - includeFiles?: boolean; - /** - * Optional configuration to turn off fetching metadata for spreadsheets. - */ - includeSpreadsheets?: boolean; - /** - * Optional configuration to turn off fetching metadata for worksheets. - */ - includeWorksheets?: boolean; - /** - * Optional configuration to soft delete directories in OpenMetadata if the source - * directories are deleted. Also, if the directory is deleted, all the associated entities - * like files, spreadsheets, worksheets, lineage, etc., with that directory will be deleted - */ - markDeletedDirectories?: boolean; - /** - * Optional configuration to soft delete files in OpenMetadata if the source files are - * deleted. Also, if the file is deleted, all the associated entities like lineage, etc., - * with that file will be deleted - */ - markDeletedFiles?: boolean; - /** - * Optional configuration to soft delete spreadsheets in OpenMetadata if the source - * spreadsheets are deleted. Also, if the spreadsheet is deleted, all the associated - * entities like worksheets, lineage, etc., with that spreadsheet will be deleted - */ - markDeletedSpreadsheets?: boolean; - /** - * Optional configuration to soft delete worksheets in OpenMetadata if the source worksheets - * are deleted. Also, if the worksheet is deleted, all the associated entities like lineage, - * etc., with that worksheet will be deleted - */ - markDeletedWorksheets?: boolean; - /** - * Regex to only include/exclude spreadsheets that matches the pattern. - */ - spreadsheetFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude worksheets that matches the pattern. - */ - worksheetFilterPattern?: FilterPattern; - /** - * Enable the 'Include Index Template' toggle to manage the ingestion of index template data. - */ - includeIndexTemplate?: boolean; - /** - * Optional configuration to turn off fetching sample data for search index. - */ - includeSampleData?: boolean; - /** - * Optional configuration to soft delete search indexes in OpenMetadata if the source search - * indexes are deleted. Also, if the search index is deleted, all the associated entities - * like lineage, etc., with that search index will be deleted - */ - markDeletedSearchIndexes?: boolean; - /** - * No. of records of sample data we want to ingest. - */ - sampleSize?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Fully qualified name of the entity to be tested, if we're working with a basic suite. - */ - entityFullyQualifiedName?: string; - /** - * Service connections to be used for the logical test suite. - */ - serviceConnections?: ServiceConnections[]; - /** - * List of test cases to be executed on the entity. If null, all test cases will be executed. - */ - testCases?: string[]; - /** - * Maximum number of events entities in a batch (Default 1000). - */ - batchSize?: number; - /** - * Certificate path to be added in configuration. The path should be local in the Ingestion - * Container. - */ - caCerts?: string; - recreateIndex?: boolean; - /** - * Region name. Required when using AWS Credentials. - */ - regionName?: string; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Connection Timeout - */ - timeout?: number; - /** - * Indicates whether to use aws credentials when connecting to OpenSearch in AWS. - */ - useAwsCredentials?: boolean; - /** - * Indicates whether to use SSL when connecting to ElasticSearch. By default, we will ignore - * SSL settings. - */ - useSSL?: boolean; - /** - * Indicates whether to verify certificates when using SSL connection to ElasticSearch. - * Ignored by default. Is set to true, make sure to send the certificates in the property - * `CA Certificates`. - */ - verifyCerts?: boolean; - /** - * Custom OpenMetadata Classification name for dbt tags. - */ - dbtClassificationName?: string; - /** - * Available sources to fetch DBT catalog and manifest files. - */ - dbtConfigSource?: DBTConfigurationSource; - /** - * Optional configuration to update the description from DBT or not - */ - dbtUpdateDescriptions?: boolean; - /** - * Optional configuration to update the owners from DBT or not - */ - dbtUpdateOwners?: boolean; - /** - * Optional configuration to search across databases for tables or not - */ - searchAcrossDatabases?: boolean; - /** - * Regex to only fetch tags that matches the pattern. - */ - tagFilterPattern?: FilterPattern; - /** - * Application configuration - */ - appConfig?: any[] | boolean | number | null | CollateAIAppConfig | string; - /** - * Application private configuration - */ - appPrivateConfig?: PrivateConfig; - /** - * Source Python Class Name to run the application - */ - sourcePythonClass?: string; - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete api collections in OpenMetadata if the source - * collections are deleted. Also, if the collection is deleted, all the associated entities - * like endpoints, etc., with that collection will be deleted - */ - markDeletedApiCollections?: boolean; - /** - * Optional value of the ingestion runner name responsible for running the workflow - */ - ingestionRunner?: string; - /** - * List of operations to be performed on the service - */ - operations?: Operation[]; - /** - * Service to be modified - */ - service?: EntityReference; -} - -/** - * Configuration for the CollateAI External Application. - * - * Configuration for the Automator External Application. - * - * This schema defines the Slack App Token Configuration - * - * No configuration needed to instantiate the Data Insights Pipeline. The logic is handled - * in the backend. - * - * Search Indexing App. - * - * Configuration for the Collate AI Quality Agent. - * - * Configuration for the AutoPilot Application. - */ -export interface CollateAIAppConfig { - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - filter?: string; - /** - * Patch the description if it is empty, instead of raising a suggestion - */ - patchIfEmpty?: boolean; - /** - * Application Type - */ - type?: CollateAIAppConfigType; - /** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - */ - actions?: Action[]; - /** - * Entities selected to run the automation. - */ - resources?: Resource; - /** - * Bot Token - */ - botToken?: string; - /** - * User Token - */ - userToken?: string; - backfillConfiguration?: BackfillConfiguration; - /** - * Maximum number of events processed at a time (Default 100). - * - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - moduleConfiguration?: ModuleConfiguration; - /** - * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property - * Type and are facing errors. Bear in mind that recreating the index will delete your - * DataAssets and a backfill will be needed. - */ - recreateDataAssetsIndex?: boolean; - sendToAdmins?: boolean; - sendToTeams?: boolean; - /** - * Enable automatic performance tuning based on cluster capabilities and database entity - * count - */ - autoTune?: boolean; - /** - * Number of threads to use for reindexing - */ - consumerThreads?: number; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Initial backoff time in milliseconds - */ - initialBackoff?: number; - /** - * Maximum backoff time in milliseconds - */ - maxBackoff?: number; - /** - * Maximum number of concurrent requests to the search index - */ - maxConcurrentRequests?: number; - /** - * Maximum number of retries for a failed request - */ - maxRetries?: number; - /** - * Maximum number of events sent in a batch (Default 100). - */ - payLoadSize?: number; - /** - * Number of threads to use for reindexing - */ - producerThreads?: number; - /** - * Queue Size to user internally for reindexing. - */ - queueSize?: number; - /** - * This schema publisher run modes. - */ - recreateIndex?: boolean; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Whether the suggested tests should be active or not upon suggestion - * - * Whether the AutoPilot Workflow should be active or not. - */ - active?: boolean; - /** - * Enter the retention period for Activity Threads of type = 'Conversation' records in days - * (e.g., 30 for one month, 60 for two months). - */ - activityThreadsRetentionPeriod?: number; - /** - * Enter the retention period for change event records in days (e.g., 7 for one week, 30 for - * one month). - */ - changeEventRetentionPeriod?: number; - /** - * Service Entity Link for which to trigger the application. - */ - entityLink?: string; - [property: string]: any; -} - -/** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - * - * Apply Classification Tags to the selected assets. - * - * Remove Classification Tags Action Type - * - * Apply Glossary Terms to the selected assets. - * - * Remove Glossary Terms Action Type - * - * Add domains to the selected assets. - * - * Remove domains from the selected assets. - * - * Apply Tags to the selected assets. - * - * Add a Custom Property to the selected assets. - * - * Remove Owner Action Type - * - * Add an owner to the selected assets. - * - * Add Test Cases to the selected assets. - * - * Remove Test Cases Action Type - * - * Add owners to the selected assets. - * - * Remove Custom Properties Action Type - * - * Add a Data Product to the selected assets. - * - * Remove a Data Product to the selected assets. - * - * Propagate description, tags and glossary terms via lineage - * - * ML Tagging action configuration for external automator. - */ -export interface Action { - /** - * Apply tags to the children of the selected assets that match the criteria. E.g., columns, - * tasks, topic fields,... - * - * Remove tags from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply terms to the children of the selected assets that match the criteria. E.g., - * columns, tasks, topic fields,... - * - * Remove terms from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply the description to the children of the selected assets that match the criteria. - * E.g., columns, tasks, topic fields,... - * - * Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Add tests to the selected table columns - * - * Remove tests to the selected table columns - */ - applyToChildren?: string[]; - /** - * Update tags even if they are already defined in the asset. By default, incoming tags are - * merged with the existing ones. - * - * Update terms even if they are already defined in the asset. By default, incoming terms - * are merged with the existing ones. - * - * Update the domains even if they are defined in the asset. By default, we will only apply - * the domains to assets without domains. - * - * Update the description even if they are already defined in the asset. By default, we'll - * only add the descriptions to assets without the description set. - * - * Update the Custom Property even if it is defined in the asset. By default, we will only - * apply the owners to assets without the given Custom Property informed. - * - * Update the tier even if it is defined in the asset. By default, we will only apply the - * tier to assets without tier. - * - * Update the test even if it is defined in the asset. By default, we will only apply the - * test to assets without the existing test already existing. - * - * Update the owners even if it is defined in the asset. By default, we will only apply the - * owners to assets without owner. - * - * Update the Data Product even if the asset belongs to a different Domain. By default, we - * will only add the Data Product if the asset has no Domain, or it belongs to the same - * domain as the Data Product. - * - * Update descriptions, tags and Glossary Terms via lineage even if they are already defined - * in the asset. By default, descriptions are only updated if they are not already defined - * in the asset, and incoming tags are merged with the existing ones. - */ - overwriteMetadata?: boolean; - /** - * Classification Tags to apply (source must be 'Classification') - * - * Classification Tags to remove (source must be 'Classification') - */ - tags?: TierElement[]; - /** - * Application Type - */ - type: ActionType; - /** - * Remove tags from all the children and parent of the selected assets. - * - * Remove terms from all the children and parent of the selected assets. - * - * Remove descriptions from all the children and parent of the selected assets. - */ - applyToAll?: boolean; - /** - * Remove tags by its label type - * - * Remove terms by its label type - */ - labels?: LabelElement[]; - /** - * Glossary Terms to apply - * - * Glossary Terms to remove - */ - terms?: TierElement[]; - /** - * Domains to apply - */ - domains?: EntityReference[]; - /** - * Description to apply - */ - description?: string; - /** - * Owners to apply - * - * Custom Properties keys to remove - */ - customProperties?: any; - /** - * tier to apply - */ - tier?: TierElement; - /** - * Test Cases to apply - */ - testCases?: TestCaseDefinitions[]; - /** - * Remove all test cases - */ - removeAll?: boolean; - /** - * Test Cases to remove - */ - testCaseDefinitions?: string[]; - /** - * Owners to apply - */ - owners?: EntityReference[]; - /** - * Data Products to apply - * - * Data Products to remove - */ - dataProducts?: EntityReference[]; - /** - * Propagate the metadata to columns via column-level lineage. - */ - propagateColumnLevel?: boolean; - /** - * Propagate description through lineage - */ - propagateDescription?: boolean; - /** - * Propagate domains from the parent through lineage - */ - propagateDomains?: boolean; - /** - * Propagate glossary terms through lineage - */ - propagateGlossaryTerms?: boolean; - /** - * Propagate owner from the parent - */ - propagateOwner?: boolean; - /** - * Propagate the metadata to the parents (e.g., tables) via lineage. - */ - propagateParent?: boolean; - /** - * Propagate tags through lineage - */ - propagateTags?: boolean; - /** - * Propagate tier from the parent - */ - propagateTier?: boolean; - /** - * Number of levels to propagate lineage. If not set, it will propagate to all levels. - */ - propagationDepth?: number; - /** - * List of configurations to stop propagation based on conditions - */ - propagationStopConfigs?: PropagationStopConfig[]; -} - -/** - * Remove tags by its label type - * - * Remove terms by its label type - */ -export enum LabelElement { - Automated = "Automated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Configuration to stop lineage propagation based on conditions - */ -export interface PropagationStopConfig { - /** - * The metadata attribute to check for stopping propagation - */ - metadataAttribute: MetadataAttribute; - /** - * List of attribute values that will stop propagation when any of them is matched - */ - value: Array; -} - -/** - * The metadata attribute to check for stopping propagation - */ -export enum MetadataAttribute { - Description = "description", - Domains = "domains", - GlossaryTerms = "glossaryTerms", - Owner = "owner", - Tags = "tags", - Tier = "tier", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - * - * Domains the asset belongs to. When not set, the asset inherits the domain from the parent - * it belongs to. - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * The ingestion agent responsible for executing the ingestion pipeline. - * - * The processing engine responsible for executing the ingestion pipeline logic. - * - * Link to the service (such as database, messaging, storage services, etc. for which this - * ingestion pipeline ingests the metadata from. - * - * Service to be modified - */ -export interface TagLabel { - /** - * Description for the tag label. - * - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this tag. - * - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Link to the tag resource. - * - * Link to the entity resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType?: LabelTypeEnum; - /** - * Name of the tag or glossary term. - * - * Name of the entity instance. - */ - name?: string; - /** - * Label is from Tags or Glossary. - */ - source?: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state?: State; - style?: Style; - tagFQN?: string; - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id?: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type?: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelTypeEnum { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - */ -export interface TierElement { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelTypeEnum; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Minimum set of requirements to get a Test Case request ready - */ -export interface TestCaseDefinitions { - /** - * Compute the passed and failed row count for the test case. - */ - computePassedFailedRowCount?: boolean; - parameterValues?: TestCaseParameterValue[]; - /** - * Tags to apply - */ - tags?: TierElement[]; - /** - * Fully qualified name of the test definition. - */ - testDefinition?: string; - /** - * If the test definition supports it, use dynamic assertion to evaluate the test case. - */ - useDynamicAssertion?: boolean; - [property: string]: any; -} - -/** - * This schema defines the parameter values that can be passed for a Test Case. - */ -export interface TestCaseParameterValue { - /** - * name of the parameter. Must match the parameter names in testCaseParameterDefinition - */ - name?: string; - /** - * value to be passed for the Parameters. These are input from Users. We capture this in - * string and convert during the runtime. - */ - value?: string; - [property: string]: any; -} - -/** - * Application Type - * - * Add Tags action type. - * - * Remove Classification Tags Action Type. - * - * Add Terms action type. - * - * Remove Terms Action Type. - * - * Add Domain Action Type. - * - * Remove Domain Action Type - * - * Add Description Action Type. - * - * Add Custom Properties Action Type. - * - * Remove Description Action Type - * - * Add Tier Action Type. - * - * Remove Tier Action Type - * - * Add Test Case Action Type. - * - * Remove Test Case Action Type - * - * Add Owner Action Type. - * - * Remove Owner Action Type - * - * Remove Custom Properties Action Type. - * - * Add Data Products Action Type. - * - * Remove Data Products Action Type. - * - * Lineage propagation action type. - * - * ML PII Tagging action type. - */ -export enum ActionType { - AddCustomPropertiesAction = "AddCustomPropertiesAction", - AddDataProductAction = "AddDataProductAction", - AddDescriptionAction = "AddDescriptionAction", - AddDomainAction = "AddDomainAction", - AddOwnerAction = "AddOwnerAction", - AddTagsAction = "AddTagsAction", - AddTermsAction = "AddTermsAction", - AddTestCaseAction = "AddTestCaseAction", - AddTierAction = "AddTierAction", - LineagePropagationAction = "LineagePropagationAction", - MLTaggingAction = "MLTaggingAction", - RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", - RemoveDataProductAction = "RemoveDataProductAction", - RemoveDescriptionAction = "RemoveDescriptionAction", - RemoveDomainAction = "RemoveDomainAction", - RemoveOwnerAction = "RemoveOwnerAction", - RemoveTagsAction = "RemoveTagsAction", - RemoveTermsAction = "RemoveTermsAction", - RemoveTestCaseAction = "RemoveTestCaseAction", - RemoveTierAction = "RemoveTierAction", -} - -/** - * Backfill Configuration - */ -export interface BackfillConfiguration { - /** - * Enable Backfill for the configured dates - */ - enabled?: boolean; - /** - * Date for which the backfill will end - */ - endDate?: Date; - /** - * Date from which to start the backfill - */ - startDate?: Date; - [property: string]: any; -} - -/** - * Different Module Configurations - */ -export interface ModuleConfiguration { - /** - * App Analytics Module configuration - */ - appAnalytics: AppAnalyticsConfig; - /** - * Cost Analysis Insights Module configuration - */ - costAnalysis: CostAnalysisConfig; - /** - * Data Assets Insights Module configuration - */ - dataAssets: DataAssetsConfig; - /** - * Data Quality Insights Module configuration - */ - dataQuality: DataQualityConfig; -} - -/** - * App Analytics Module configuration - */ -export interface AppAnalyticsConfig { - /** - * If Enabled, App Analytics insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Cost Analysis Insights Module configuration - */ -export interface CostAnalysisConfig { - /** - * If Enabled, Cost Analysis insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Data Assets Insights Module configuration - */ -export interface DataAssetsConfig { - /** - * If Enabled, Data Asset insights will be populated when the App runs. - */ - enabled: boolean; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Defines the number of days the Data Assets Insights information will be kept. After it - * they will be deleted. - */ - retention?: number; - serviceFilter?: ServiceFilter; -} - -export interface ServiceFilter { - serviceName?: string; - serviceType?: string; -} - -/** - * Data Quality Insights Module configuration - */ -export interface DataQualityConfig { - /** - * If Enabled, Data Quality insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Entities selected to run the automation. - */ -export interface Resource { - /** - * Filter JSON tree to be used for rendering the filters in the UI. This comes from - * Immutable Tree type of react-awesome-query-builder. - */ - filterJsonTree?: string; - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - queryFilter?: string; - /** - * Type of the entity. E.g., 'table', 'chart',... - */ - type?: string[]; - [property: string]: any; -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Application Type - * - * Application type. - */ -export enum CollateAIAppConfigType { - AutoPilotApplication = "AutoPilotApplication", - Automator = "Automator", - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - DataInsights = "DataInsights", - DataInsightsReport = "DataInsightsReport", - SearchIndexing = "SearchIndexing", -} - -/** - * Application private configuration - * - * Private Configuration for the CollateAI External Application. - */ -export interface PrivateConfig { - /** - * Collate Server public URL. WAII will use this information to interact with the server. - * E.g., https://sandbox.getcollate.io - */ - collateURL?: string; - /** - * Limits for the CollateAI Application. - */ - limits?: AppLimitsConfig; - /** - * WAII API Token - */ - token?: string; - /** - * WAII API host URL - */ - waiiInstance?: string; - [property: string]: any; -} - -/** - * Limits for the CollateAI Application. - * - * Private Configuration for the App Limits. - */ -export interface AppLimitsConfig { - /** - * The records of the limits. - */ - actions: { [key: string]: number }; - /** - * The start of this limit cycle. - */ - billingCycleStart: Date; -} - -/** - * Available sources to fetch DBT catalog and manifest files. - * - * dbt Cloud configuration. - * - * DBT Catalog, Manifest and Run Results file path config. - * - * DBT Catalog, Manifest and Run Results HTTP path configuration. - * - * DBT Catalog, Manifest and Run Results files in S3 bucket. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in GCS storage. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in Azure bucket. We will search for - * catalog.json, manifest.json and run_results.json. - */ -export interface DBTConfigurationSource { - /** - * dbt cloud account Id - */ - dbtCloudAccountId?: string; - /** - * dbt cloud account authentication token - */ - dbtCloudAuthToken?: string; - /** - * dbt cloud job id. - */ - dbtCloudJobId?: string; - /** - * In case of multiple projects in a dbt cloud account, specify the project's id from which - * you want to extract the dbt run artifacts - */ - dbtCloudProjectId?: string; - /** - * URL to connect to your dbt cloud instance. E.g., https://cloud.getdbt.com or - * https://emea.dbt.com/ - */ - dbtCloudUrl?: string; - /** - * dbt Configuration type - */ - dbtConfigType: DbtConfigType; - /** - * DBT catalog file path to extract dbt models with their column schemas. - */ - dbtCatalogFilePath?: string; - /** - * DBT manifest file path to extract dbt models and associate with tables. - */ - dbtManifestFilePath?: string; - /** - * DBT run results file path to extract the test results information. - */ - dbtRunResultsFilePath?: string; - /** - * DBT sources file path to extract the freshness test result. - */ - dbtSourcesFilePath?: string; - /** - * DBT catalog http file path to extract dbt models with their column schemas. - */ - dbtCatalogHttpPath?: string; - /** - * DBT manifest http file path to extract dbt models and associate with tables. - */ - dbtManifestHttpPath?: string; - /** - * DBT run results http file path to extract the test results information. - */ - dbtRunResultsHttpPath?: string; - /** - * DBT sources http file path to extract freshness test results information. - */ - dbtSourcesHttpPath?: string; - /** - * Details of the bucket where the dbt files are stored - */ - dbtPrefixConfig?: DBTPrefixConfig; - dbtSecurityConfig?: DbtSecurityConfigClass; -} - -/** - * dbt Configuration type - */ -export enum DbtConfigType { - Azure = "azure", - Cloud = "cloud", - Gcs = "gcs", - HTTP = "http", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the dbt files are stored - */ -export interface DBTPrefixConfig { - /** - * Name of the bucket where the dbt files are stored - */ - dbtBucketName?: string; - /** - * Path of the folder where the dbt files are stored - */ - dbtObjectPrefix?: string; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - * - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - */ -export interface DbtSecurityConfigClass { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Use incremental Metadata extraction after the first execution. This is commonly done by - * getting the changes from Audit tables on the supporting databases. - */ -export interface IncrementalMetadataExtractionConfiguration { - /** - * If True, enables Metadata Extraction to be incremental - */ - enabled: boolean; - /** - * Number os days to search back for a successful pipeline run. The timestamp of the last - * found successful pipeline run will be used as a base to search for updated entities. - */ - lookbackDays?: number; - /** - * Number of days to add to the last successful pipeline run timestamp to search for updated - * entities. - */ - safetyMarginDays?: number; -} - -/** - * Details required to generate Lineage - */ -export interface LineageInformation { - /** - * List of service path prefixes for lineage matching. Supported formats: DBServiceName, - * DBServiceName.DatabaseName, DBServiceName.DatabaseName.SchemaName, or - * DBServiceName.DatabaseName.SchemaName.TableName - */ - dbServicePrefixes?: string[]; - /** - * List of Database Service Names for creation of lineage - */ - dbServiceNames?: string[]; - /** - * List of Storage Service Names for creation of lineage - */ - storageServiceNames?: string[]; - [property: string]: any; -} - -/** - * Operation to be performed on the entity - */ -export interface Operation { - /** - * Entity to be modified - */ - entityLink: string; - /** - * The id of the operation - */ - id: string; - /** - * The configuration for the operation to be applied - */ - parameters: ReverseIngestionConfig; - /** - * Templated SQL command to be used for the operation. Context parameters will be populated - * based on the event type. - */ - SQLTemplate?: string; - /** - * Type of operation to perform - */ - type: OperationType; -} - -/** - * The configuration for the operation to be applied - * - * Configuration for updating descriptions - * - * Configuration for updating owners - * - * Configuration for updating tags - */ -export interface ReverseIngestionConfig { - /** - * New description of the service - */ - newDescription?: string; - /** - * Previous description of the service - */ - previousDescription?: string; - /** - * Added owners to be applied - */ - addedOwners?: EntityReference[]; - /** - * Removed owners from the entity - */ - removedOwners?: EntityReference[]; - /** - * Added tags to be applied - */ - addedTags?: TierElement[]; - /** - * Removed tags of the entity - */ - removedTags?: TierElement[]; -} - -/** - * Type of operation to perform - */ -export enum OperationType { - UpdateDescription = "UPDATE_DESCRIPTION", - UpdateOwner = "UPDATE_OWNER", - UpdateTags = "UPDATE_TAGS", -} - -/** - * Processing Engine Configuration. If not provided, the Native Engine will be used by - * default. - * - * Configuration for the native metadata ingestion engine - * - * This schema defines the configuration for a Spark Engine runner. - */ -export interface ProcessingEngine { - /** - * The type of the engine configuration - */ - type: ProcessingEngineType; - config?: Config; - /** - * Spark Connect Remote URL. - */ - remote?: string; -} - -export interface Config { - /** - * Additional Spark configuration properties as key-value pairs. - */ - extraConfig?: { [key: string]: any }; - /** - * Temporary path to store the data. - */ - tempPath?: string; - [property: string]: any; -} - -/** - * The type of the engine configuration - */ -export enum ProcessingEngineType { - Native = "Native", - Spark = "Spark", -} - -/** - * Type of Profile Sample (percentage or rows) - */ -export enum ProfileSampleType { - Percentage = "PERCENTAGE", - Rows = "ROWS", -} - -/** - * Type of Sampling Method (BERNOULLI or SYSTEM) - */ -export enum SamplingMethodType { - Bernoulli = "BERNOULLI", - System = "SYSTEM", -} - -/** - * Service connections available for the logical test suite. - */ -export interface ServiceConnections { - /** - * Connection configuration for the source. ex: mysql , tableau connection. - */ - serviceConnection: ServiceConnection; - serviceName: string; -} - -/** - * Connection configuration for the source. ex: mysql , tableau connection. - * - * Supported services - * - * API Service Connection. - * - * Dashboard Connection. - * - * Database Connection. - * - * Metadata Service Connection. - * - * Pipeline Connection. - * - * MlModel Connection. - * - * Storage Connection. - * - * search Connection. - * - * Security Connection. - * - * Drive Connection. - */ -export interface ServiceConnection { - config?: ConfigObject; -} - -/** - * REST Connection Config - * - * Looker Connection Config - * - * Metabase Connection Config - * - * PowerBI Connection Config - * - * PowerBIReportServer Connection Config - * - * Redash Connection Config - * - * Superset Connection Config - * - * Tableau Connection Config - * - * Mode Connection Config - * - * Custom Dashboard Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Domo Dashboard Connection Config - * - * QuickSight Connection Config - * - * Qlik Sense Connection Config - * - * Lightdash Connection Config - * - * MicroStrategy Connection Config - * - * Qlik Cloud Connection Config - * - * Sigma Connection Config - * - * ThoughtSpot Connection Config - * - * Grafana Connection Config - * - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - * - * Kafka Connection Config - * - * Redpanda Connection Config - * - * Kinesis Connection Config - * - * Custom Messaging Service Connection to build a source that is not supported by - * OpenMetadata yet. - * - * Amundsen Connection Config - * - * Metadata to ElasticSearch Connection Config - * - * OpenMetadata Connection Config - * - * Atlas Connection Config - * - * Alation Connection Config - * - * Alation Sink Connection Config - * - * Airflow Metadata Database Connection Config - * - * Wherescape Metadata Database Connection Config - * - * SSIS Metadata Database Connection Config - * - * Glue Pipeline Connection Config - * - * Airbyte Metadata Database Connection Config - * - * Fivetran Metadata Database Connection Config - * - * Flink Metadata Connection Config - * - * Dagster Metadata Database Connection Config - * - * Nifi Metadata Pipeline Connection Config - * - * Domo Pipeline Connection Config - * - * Custom Pipeline Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Spline Metadata Database Connection Config - * - * Spark Metadata Pipeline Connection Config - * - * OpenLineage Connection Config - * - * KafkaConnect Connection Config - * - * DBTCloud Connection Config - * - * Matillion Connection - * - * Azure Data Factory Connection Config - * - * Stitch Connection - * - * Snowplow Pipeline Connection Config - * - * MlFlow Connection Config - * - * Sklearn Connection Config - * - * Custom MlModel Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * SageMaker Connection Config - * - * Google VertexAI Connection Config - * - * S3 Connection. - * - * ADLS Connection. - * - * GCS Connection. - * - * Custom Storage Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * ElasticSearch Connection. - * - * OpenSearch Connection Config - * - * Custom Search Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * Apache Ranger Connection Config - * - * Google Drive Connection Config - * - * SharePoint Connection Config - * - * Custom Drive Connection to build a source that is not supported. - */ -export interface ConfigObject { - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Documentation URL for the schema. - */ - docURL?: string; - /** - * Open API Schema URL. - */ - openAPISchemaURL?: string; - /** - * Supports Metadata Extraction. - */ - supportsMetadataExtraction?: boolean; - /** - * Generated Token to connect to OpenAPI Schema. - * - * token to connect to Qlik Cloud. - * - * Generated Token to connect to Databricks. - * - * To Connect to Dagster Cloud - * - * Generated Token to connect to DBTCloud. - * - * Token to connect to Stitch api doc - */ - token?: string; - /** - * REST API Type - * - * Service Type - * - * Custom dashboard service type - * - * Custom database service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * OpenSearch Type - * - * Custom search service type - */ - type?: PurpleType; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * User's Client ID. This user should have privileges to read all the metadata in Looker. - * - * client_id for PowerBI. - * - * Client ID for DOMO - * - * client_id for Sigma. - * - * Azure Application (client) ID for service principal authentication. - * - * Application (client) ID from Azure Active Directory - */ - clientId?: string; - /** - * User's Client Secret. - * - * clientSecret for PowerBI. - * - * clientSecret for Sigma. - * - * Azure Application client secret for service principal authentication. - * - * Application (client) secret from Azure Active Directory - */ - clientSecret?: string; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - */ - gitCredentials?: GitHubCredentials; - /** - * URL to the Looker instance. - * - * Host and Port of the Metabase instance. - * - * Dashboard URL for PowerBI service. - * - * Dashboard URL for PowerBI Report Server. - * - * URL for the Redash instance - * - * URL for the superset instance. - * - * Tableau Server url. - * - * URL for the mode instance. - * - * URL for the Qlik instance. - * - * Address for your running Lightdash instance - * - * Host and Port of the MicroStrategy instance. - * - * Host and Port of the Qlik Cloud instance. - * - * Sigma API url. - * - * ThoughtSpot instance URL. Example: https://my-company.thoughtspot.cloud - * - * URL to the Grafana instance. - * - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - * - * Host and port of the Amundsen Neo4j Connection. This expect a URI format like: - * bolt://localhost:7687. - * - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - * - * Host and port of the Atlas service. - * - * Host and port of the Alation service. - * - * Pipeline Service Management/UI URI. - * - * Pipeline Service Management/UI URL. - * - * Spline REST Server Host & Port. - * - * KafkaConnect Service Management/UI URI. - * - * Host and port of the Stitch API host - * - * Host and port of the ElasticSearch service. - * - * Host and port of the OpenSearch service. - * - * Apache Ranger Admin URL. - */ - hostPort?: string; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * API token to connect to Metabase. Use this instead of username/password for token-based - * authentication. - * - * API key of the redash instance to access. - * - * The personal access token you can generate in the Lightdash app under the user settings - * - * Service Account Token to authenticate to the Grafana APIs. Use Service Account Tokens - * (format: glsa_xxxx) for authentication. Legacy API Keys are no longer supported by - * Grafana as of January 2025. Both self-hosted and Grafana Cloud are supported. Requires - * Admin role for full metadata extraction. - * - * API key to authenticate with the SAP ERP APIs. - * - * Fivetran API Secret. - * - * API Key for Snowplow Console API - */ - apiKey?: string; - /** - * Password to connect to Metabase. Required for basic authentication. - * - * Password to connect to PowerBI report server. - * - * Password to connect to MicroStrategy. - * - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to the Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - * - * password to connect to the Amundsen Neo4j Connection. - * - * password to connect to the Atlas. - * - * Password to connect to Airbyte. - */ - password?: string; - /** - * Username to connect to Metabase. Required for basic authentication. - * - * Username to connect to PowerBI report server. - * - * Username for Redash - * - * Username to connect to MicroStrategy. This user should have privileges to read all the - * metadata in MicroStrategy. - * - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to the Salesforce. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - * - * username to connect to the Amundsen Neo4j Connection. - * - * username to connect to the Atlas. This user should have privileges to read all the - * metadata in Atlas. - * - * Username to connect to Airbyte. - */ - username?: string; - /** - * Authority URI for the PowerBI service. - */ - authorityURI?: string; - /** - * Display Table Name from source instead of renamed table name for datamodel tables - */ - displayTableNameFromSource?: boolean; - /** - * Entity Limit set here will be used to paginate the PowerBi APIs - */ - pagination_entity_per_page?: number; - /** - * Source to get the .pbit files to extract lineage information - */ - pbitFilesSource?: PowerBIPbitFilesSource; - /** - * PowerBI secrets. - */ - scope?: string[]; - /** - * Tenant ID for PowerBI. - * - * Azure Directory (tenant) ID for service principal authentication. - * - * Directory (tenant) ID from Azure Active Directory - */ - tenantId?: string; - /** - * Fetch the PowerBI metadata using admin APIs - */ - useAdminApis?: boolean; - /** - * Web Portal Virtual Directory Name. - */ - webPortalVirtualDirectory?: string; - /** - * Version of the Redash instance - */ - redashVersion?: string; - /** - * Choose between API or database connection fetch metadata from superset. - * - * Choose between Database connection or HDB User Store connection. - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Matillion Auth Configuration - */ - connection?: ConfigConnection; - /** - * Tableau API version. If not provided, the version will be used from the tableau server. - * - * Sigma API version. - * - * ThoughtSpot API version to use - * - * OpenMetadata server API version to use. - * - * Airbyte API version. - */ - apiVersion?: string; - /** - * Types of methods used to authenticate to the tableau instance - * - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - * - * Types of methods used to authenticate to the alation instance - * - * Authentication type to connect to Apache Ranger. - */ - authType?: AuthenticationTypeForTableau | NoConfigAuthenticationTypes; - /** - * Pagination limit used while querying the tableau metadata API for getting data sources - * - * Pagination limit used while querying the SAP ERP API for fetching the entities - * - * Pagination limit used for Alation APIs pagination - */ - paginationLimit?: number; - /** - * Proxy URL for the tableau server. If not provided, the hostPort will be used. This is - * used to generate the dashboard & Chart URL. - */ - proxyURL?: string; - /** - * Tableau Site Name. - * - * SharePoint site name - */ - siteName?: string; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; - /** - * Boolean marking if we need to verify the SSL certs for Grafana. Default to True. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Boolean marking if we need to verify the SSL certs for KafkaConnect REST API. True by - * default. - */ - verifySSL?: boolean | VerifySSL; - /** - * Access Token for Mode Dashboard - * - * Access token to connect to DOMO - */ - accessToken?: string; - /** - * Access Token Password for Mode Dashboard - */ - accessTokenPassword?: string; - /** - * Filter query parameter for some of the Mode API calls - */ - filterQueryParam?: string; - /** - * Mode Workspace Name - */ - workspaceName?: string; - connectionOptions?: { [key: string]: string }; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - * - * Secret token to connect to DOMO - */ - secretToken?: string; - /** - * AWS Account ID - */ - awsAccountId?: string; - awsConfig?: AWSCredentials; - /** - * The authentication method that the user uses to sign in. - */ - identityType?: IdentityType; - /** - * The Amazon QuickSight namespace that contains the dashboard IDs in this request ( To be - * provided when identityType is `ANONYMOUS` ) - */ - namespace?: string; - certificates?: QlikCertificatesBy; - /** - * Qlik Sense Base URL, used for genrating dashboard & chat url - */ - displayUrl?: string; - /** - * User Directory. - */ - userDirectory?: string; - /** - * User ID. - */ - userId?: string; - /** - * Validate Host Name - */ - validateHostName?: boolean; - /** - * The Project UUID for your Lightdash instance - */ - projectUUID?: string; - /** - * Use if your Lightdash instance is behind a proxy like (Cloud IAP) - */ - proxyAuthentication?: string; - /** - * The Space UUID for your Lightdash instance - */ - spaceUUID?: string; - /** - * Login Mode for Microstrategy's REST API connection. You can authenticate with one of the - * following authentication modes: `Standard (1)`, `Anonymous (8)`. Default will be - * `Standard (1)`. If you're using demo account for Microstrategy, it will be needed to - * authenticate through loginMode `8`. - */ - loginMode?: string; - /** - * MicroStrategy Project Name - * - * Project name to create the refreshToken. Can be anything - */ - projectName?: string; - /** - * Space types of Qlik Cloud to filter the dashboards ingested into the platform. - */ - spaceTypes?: SpaceType[]; - /** - * ThoughtSpot authentication configuration - */ - authentication?: Authenticationation; - /** - * Org ID for multi-tenant ThoughtSpot instances. This is applicable for ThoughtSpot Cloud - * only. - */ - orgId?: string; - /** - * Page size for pagination in API requests. Default is 100. - */ - pageSize?: number; - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - * - * Azure Credentials - * - * GCP Credentials for Google Drive API - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - * - * Connection timeout in seconds. - */ - connectionTimeout?: number | number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - * - * Available sources to fetch metadata. - */ - configSource?: DeltaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - */ - auth?: AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - * - * Snowplow BDP Organization ID - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - /** - * basic.auth.user.info schema registry config property, Client HTTP credentials in the form - * of username:password. - */ - basicAuthUserInfo?: string; - /** - * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - * - * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - */ - bootstrapServers?: string; - /** - * Confluent Kafka Consumer Config. From - * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - * - * Confluent Redpanda Consumer Config - */ - consumerConfig?: { [key: string]: any }; - /** - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - */ - consumerConfigSSL?: ConsumerConfigSSLClass; - /** - * sasl.mechanism Consumer Config property - */ - saslMechanism?: SaslMechanismType; - /** - * sasl.password consumer config property - */ - saslPassword?: string; - /** - * sasl.username consumer config property - */ - saslUsername?: string; - /** - * Confluent Kafka Schema Registry Config. From - * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient - * - * Confluent Redpanda Schema Registry Config. - */ - schemaRegistryConfig?: { [key: string]: any }; - /** - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ - schemaRegistrySSL?: ConsumerConfigSSLClass; - /** - * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get - * topic schema from registry. - */ - schemaRegistryTopicSuffixName?: string; - /** - * Confluent Kafka Schema Registry URL. - * - * Confluent Redpanda Schema Registry URL. - */ - schemaRegistryURL?: string; - /** - * security.protocol consumer config property - * - * Kafka security protocol config - */ - securityProtocol?: KafkaSecurityProtocol; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Enable encryption for the Amundsen Neo4j Connection. - */ - encrypted?: boolean; - /** - * Maximum connection lifetime for the Amundsen Neo4j Connection. - */ - maxConnectionLifeTime?: number; - /** - * Enable SSL validation for the Amundsen Neo4j Connection. - */ - validateSSL?: boolean; - /** - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - /** - * List of entities that you need to reindex - */ - entities?: string[]; - recreateIndex?: boolean; - runMode?: RunMode; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ConfigElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * service type of the data source. - */ - databaseServiceName?: string[]; - /** - * Name of the Entity Type available in Atlas. - */ - entity_type?: string; - /** - * service type of the messaging source - * - * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. - * e.g. local_kafka - */ - messagingServiceName?: string[] | string; - /** - * Custom OpenMetadata Classification name for alation tags. - */ - alationTagClassificationName?: string; - /** - * Specifies if hidden datasources should be included while ingesting. - */ - includeHiddenDatasources?: boolean; - /** - * Specifies if undeployed datasources should be included while ingesting. - */ - includeUndeployedDatasources?: boolean; - /** - * Specifies if Dashboards are to be ingested while running the ingestion job. - */ - ingestDashboards?: boolean; - /** - * Specifies if Datasources are to be ingested while running the ingestion job. - */ - ingestDatasources?: boolean; - /** - * Specifies if Domains are to be ingested while running the ingestion job. - */ - ingestDomains?: boolean; - /** - * Specifies if Knowledge Articles are to be ingested while running the ingestion job. - */ - ingestKnowledgeArticles?: boolean; - /** - * Specifies if Users and Groups are to be ingested while running the ingestion job. - */ - ingestUsersAndGroups?: boolean; - datasourceLinks?: { [key: string]: string }; - /** - * Pipeline Service Number Of Status - */ - numberOfStatus?: number; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Underlying database connection - */ - databaseConnection?: DatabaseConnectionClass; - /** - * Underlying storage connection - */ - packageConnection?: S3Connection | string; - /** - * Fivetran API Secret. - */ - apiSecret?: string; - /** - * Fivetran API Limit For Pagination. - */ - limit?: number; - /** - * URL to the Dagster instance - * - * DBT cloud Access URL. - */ - host?: string; - /** - * Connection Time Limit Between OM and Dagster Graphql API in second - */ - timeout?: number; - /** - * We support username/password or client certificate authentication - */ - nifiConfig?: NifiCredentialsConfiguration; - /** - * Spline UI Host & Port. - */ - uiHostPort?: string; - /** - * service type of the messaging source - */ - brokersUrl?: string; - /** - * consumer group name - */ - consumerGroupName?: string; - /** - * initial Kafka consumer offset - */ - consumerOffsets?: InitialConsumerOffsets; - /** - * max allowed wait time - */ - poolTimeout?: number; - /** - * SASL Configuration details. - */ - saslConfig?: SASLClientConfig; - /** - * max allowed inactivity time - */ - sessionTimeout?: number; - /** - * topic from where Open lineage events will be pulled - */ - topicName?: string; - /** - * We support username/password or No Authentication - */ - KafkaConnectConfig?: UsernamePasswordAuthentication; - /** - * ID of your DBT cloud account - */ - accountId?: string; - /** - * DBT cloud Metadata API URL. - */ - discoveryAPI?: string; - /** - * List of IDs of your DBT cloud jobs seperated by comma `,` - */ - jobIds?: string[]; - /** - * Number of runs to fetch from DBT cloud - */ - numberOfRuns?: number; - /** - * List of IDs of your DBT cloud projects seperated by comma `,` - */ - projectIds?: string[]; - /** - * The name of your azure data factory. - */ - factory_name?: string; - /** - * The name of your resource group the data factory is associated with. - */ - resource_group_name?: string; - /** - * Number of days in the past to filter pipeline runs. - */ - run_filter_days?: number; - /** - * The azure subscription identifier. - */ - subscription_id?: string; - /** - * Cloud provider where Snowplow is deployed - */ - cloudProvider?: CloudProvider; - /** - * Path to pipeline configuration files for Community deployment - */ - configPath?: string; - /** - * Snowplow Console URL for BDP deployment - */ - consoleUrl?: string; - /** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - */ - deployment?: SnowplowDeployment; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Mlflow Model registry backend. E.g., - * mysql+pymysql://mlflow:password@localhost:3307/experiments - */ - registryUri?: string; - /** - * Mlflow Experiment tracking URI. E.g., http://localhost:5000 - */ - trackingUri?: string; - /** - * location/region of google cloud project - */ - location?: string; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Connection Timeout in Seconds - */ - connectionTimeoutSecs?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Email to impersonate using domain-wide delegation - */ - delegatedEmail?: string; - /** - * Specific shared drive ID to connect to - * - * SharePoint drive ID. If not provided, default document library will be used - */ - driveId?: string; - /** - * Extract metadata only for Google Sheets files - */ - includeGoogleSheets?: boolean; - /** - * Include shared/team drives in metadata extraction - */ - includeTeamDrives?: boolean; - /** - * SharePoint site URL - */ - siteUrl?: string; - [property: string]: any; -} - -/** - * We support username/password or No Authentication - * - * username/password auth - */ -export interface UsernamePasswordAuthentication { - /** - * KafkaConnect password to authenticate to the API. - */ - password?: string; - /** - * KafkaConnect user to authenticate to the API. - */ - username?: string; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Types of methods used to authenticate to the tableau instance - * - * Basic Auth Credentials - * - * Access Token Auth Credentials - * - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - * - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * API Access Token Auth Credentials - * - * Basic Auth Configuration for ElasticSearch - * - * SSL Certificates By Path - * - * AWS credentials configs. - * - * Authentication type to connect to Apache Ranger. - * - * Configuration for connecting to Ranger Basic Auth. - */ -export interface AuthenticationTypeForTableau { - /** - * Password to access the service. - * - * Password to connect to source. - * - * Elastic Search Password for Login - * - * Ranger password to authenticate to the API. - */ - password?: string; - /** - * Username to access the service. - * - * Elastic Search Username for Login - * - * Ranger user to authenticate to the API. - */ - username?: string; - /** - * Personal Access Token Name. - */ - personalAccessTokenName?: string; - /** - * Personal Access Token Secret. - */ - personalAccessTokenSecret?: string; - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; - /** - * Access Token for the API - */ - accessToken?: string; - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -/** - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * Basic Auth Credentials - * - * API Access Token Auth Credentials - */ -export interface Authenticationation { - /** - * Password to access the service. - */ - password?: string; - /** - * Username to access the service. - */ - username?: string; - /** - * Access Token for the API - */ - accessToken?: string; -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: Authentication; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum Authentication { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: Credentials | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Qlik Authentication Certificate By Values - * - * Qlik Authentication Certificate File Path - */ -export interface QlikCertificatesBy { - sslConfig?: ConsumerConfigSSLClass; - /** - * Client Certificate - */ - clientCertificate?: string; - /** - * Client Key Certificate. - */ - clientKeyCertificate?: string; - /** - * Root Certificate. - */ - rootCertificate?: string; - [property: string]: any; -} - -/** - * Cloud provider where Snowplow is deployed - */ -export enum CloudProvider { - Aws = "AWS", - Azure = "Azure", - Gcp = "GCP", -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface DeltaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConfigSourceConnection; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: DbtSecurityConfigClass; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConfigSourceConnection { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Choose between API or database connection fetch metadata from superset. - * - * Superset API Connection Config - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - * - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Lineage Backend Connection Config - * - * SQLite Database Connection Config - * - * Matillion Auth Configuration - * - * Matillion ETL Auth Config. - */ -export interface ConfigConnection { - /** - * Password for Superset. - * - * Password to connect to Hana. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to the Matillion. - */ - password?: string; - /** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ - provider?: Provider; - /** - * SSL Configuration details. - */ - sslConfig?: ConnectionSSLConfig; - /** - * Username for Superset. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to Hana. This user should have privileges to read all the metadata. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to the Matillion. This user should have privileges to read all the - * metadata in Matillion. - */ - username?: string; - verifySSL?: VerifySSL; - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - * - * Host and port of the Hana service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Matillion Host - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: ConnectionScheme; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: ConnectionType; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - supportsViewLineageExtraction?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ -export enum Provider { - DB = "db", - LDAP = "ldap", -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum ConnectionScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", - SqlitePysqlite = "sqlite+pysqlite", -} - -/** - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Configuration for OpenMetadata Server - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ -export interface ConnectionSSLConfig { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum ConnectionType { - Backend = "Backend", - MatillionETL = "MatillionETL", - Mysql = "Mysql", - Postgres = "Postgres", - SQLite = "SQLite", -} - -/** - * initial Kafka consumer offset - */ -export enum InitialConsumerOffsets { - Earliest = "earliest", - Latest = "latest", -} - -/** - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Underlying database connection - * - * Mssql Database Connection Config - */ -export interface DatabaseConnectionClass { - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Host and port of the MSSQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to MSSQL. - */ - password?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: MssqlScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: MssqlType; - /** - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - */ - username?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum MssqlScheme { - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", -} - -/** - * Service Type - * - * Service type. - */ -export enum MssqlType { - Mssql = "Mssql", -} - -/** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - * - * Snowplow deployment type - */ -export enum SnowplowDeployment { - Bdp = "BDP", - Community = "Community", -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ConfigElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - * - * Do not set any credentials. Note that credentials are required to extract .lkml views and - * their lineage. - * - * Credentials for a GitHub repository - * - * Credentials for a BitBucket repository - * - * Credentials for a Gitlab repository - */ -export interface GitHubCredentials { - repositoryName?: string; - repositoryOwner?: string; - token?: string; - /** - * Credentials Type - */ - type?: GitHubCredentialsType; - /** - * Main production branch of the repository. E.g., `main` - */ - branch?: string; -} - -/** - * Credentials Type - * - * GitHub Credentials type - * - * BitBucket Credentials type - * - * Gitlab Credentials type - */ -export enum GitHubCredentialsType { - BitBucket = "BitBucket", - GitHub = "GitHub", - Gitlab = "Gitlab", -} - -/** - * The authentication method that the user uses to sign in. - */ -export enum IdentityType { - Anonymous = "ANONYMOUS", - Iam = "IAM", - Quicksight = "QUICKSIGHT", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: ConsumerConfigSSLClass; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * We support username/password or client certificate authentication - * - * Configuration for connecting to Nifi Basic Auth. - * - * Configuration for connecting to Nifi Client Certificate Auth. - */ -export interface NifiCredentialsConfiguration { - /** - * Nifi password to authenticate to the API. - */ - password?: string; - /** - * Nifi user to authenticate to the API. - */ - username?: string; - /** - * Boolean marking if we need to verify the SSL certs for Nifi. False by default. - */ - verifySSL?: boolean; - /** - * Path to the root CA certificate - */ - certificateAuthorityPath?: string; - /** - * Path to the client certificate - */ - clientCertificatePath?: string; - /** - * Path to the client key - */ - clientkeyPath?: string; -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * S3 Connection. - */ -export interface S3Connection { - awsConfig: AWSCredentials; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: S3Type; -} - -/** - * Service Type - * - * S3 service type - */ -export enum S3Type { - S3 = "S3", -} - -/** - * Source to get the .pbit files to extract lineage information - * - * Local config source where no extra information needs to be sent. - * - * Azure storage config for pbit files - * - * GCS storage config for pbit files - * - * S3 storage config for pbit files - */ -export interface PowerBIPbitFilesSource { - /** - * Directory path for the pbit files - */ - path?: string; - /** - * pbit File Configuration type - */ - pbitFileConfigType?: PbitFileConfigType; - /** - * Path of the folder where the .pbit files will be unzipped and datamodel schema will be - * extracted - */ - pbitFilesExtractDir?: string; - prefixConfig?: BucketDetails; - securityConfig?: DbtSecurityConfigClass; -} - -/** - * pbit File Configuration type - */ -export enum PbitFileConfigType { - Azure = "azure", - Gcs = "gcs", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the .pbit files are stored - */ -export interface BucketDetails { - /** - * Name of the bucket where the .pbit files are stored - */ - bucketName?: string; - /** - * Path of the folder where the .pbit files are stored - */ - objectPrefix?: string; -} - -/** - * This schema publisher run modes. - */ -export enum RunMode { - Batch = "batch", - Stream = "stream", -} - -/** - * SASL Configuration details. - * - * SASL client configuration. - */ -export interface SASLClientConfig { - /** - * SASL security mechanism - */ - saslMechanism?: SaslMechanismType; - /** - * The SASL authentication password. - */ - saslPassword?: string; - /** - * The SASL authentication username. - */ - saslUsername?: string; -} - -/** - * sasl.mechanism Consumer Config property - * - * SASL Mechanism consumer config property - * - * SASL security mechanism - */ -export enum SaslMechanismType { - Gssapi = "GSSAPI", - Oauthbearer = "OAUTHBEARER", - Plain = "PLAIN", - ScramSHA256 = "SCRAM-SHA-256", - ScramSHA512 = "SCRAM-SHA-512", -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * security.protocol consumer config property - * - * Kafka security protocol config - */ -export enum KafkaSecurityProtocol { - Plaintext = "PLAINTEXT", - SSL = "SSL", - SaslPlaintext = "SASL_PLAINTEXT", - SaslSSL = "SASL_SSL", -} - -export enum SpaceType { - Data = "Data", - Managed = "Managed", - Personal = "Personal", - Shared = "Shared", -} - -/** - * SSL Configuration for OpenMetadata Server - * - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Config - */ -export interface SSLConfigObject { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; - /** - * SSL Certificates - */ - certificates?: SSLCertificates; - [property: string]: any; -} - -/** - * SSL Certificates - * - * SSL Configuration details. - * - * SSL Certificates By Path - * - * SSL Certificates By Values - */ -export interface SSLCertificates { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * CA Certificate Value - */ - caCertValue?: string; - /** - * Client Certificate Value - */ - clientCertValue?: string; - /** - * Private Key Value - */ - privateKeyValue?: string; - /** - * Staging Directory Path - */ - stagingDir?: string; -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * REST API Type - * - * REST API type - * - * Service Type - * - * Looker service type - * - * Metabase service type - * - * PowerBI service type - * - * PowerBIReportServer service type - * - * Redash service type - * - * Superset service type - * - * Tableau service type - * - * Mode service type - * - * Custom dashboard service type - * - * service type - * - * QuickSight service type - * - * Qlik sense service type - * - * Lightdash service type - * - * MicroStrategy service type - * - * Qlik Cloud service type - * - * Sigma service type - * - * ThoughtSpot service type - * - * Grafana service type - * - * Service type. - * - * Custom database service type - * - * Kafka service type - * - * Redpanda service type - * - * Custom messaging service type - * - * Amundsen service type - * - * Metadata to Elastic Search type - * - * OpenMetadata service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * S3 service type - * - * ADLS service type - * - * Gcs service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * ElasticSearch service type - * - * OpenSearch Type - * - * OpenSearch service type - * - * Custom search service type - * - * Apache Ranger service type - * - * Google Drive service type - * - * SharePoint service type - * - * Custom Drive service type - */ -export enum PurpleType { - Adls = "ADLS", - Airbyte = "Airbyte", - Airflow = "Airflow", - Alation = "Alation", - AlationSink = "AlationSink", - Amundsen = "Amundsen", - Athena = "Athena", - Atlas = "Atlas", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDashboard = "CustomDashboard", - CustomDatabase = "CustomDatabase", - CustomDrive = "CustomDrive", - CustomMessaging = "CustomMessaging", - CustomMlModel = "CustomMlModel", - CustomPipeline = "CustomPipeline", - CustomSearch = "CustomSearch", - CustomStorage = "CustomStorage", - DBTCloud = "DBTCloud", - Dagster = "Dagster", - DataFactory = "DataFactory", - Databricks = "Databricks", - DatabricksPipeline = "DatabricksPipeline", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDashboard = "DomoDashboard", - DomoDatabase = "DomoDatabase", - DomoPipeline = "DomoPipeline", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - ElasticSearch = "ElasticSearch", - Epic = "Epic", - Exasol = "Exasol", - Fivetran = "Fivetran", - Flink = "Flink", - Gcs = "GCS", - Glue = "Glue", - GluePipeline = "GluePipeline", - GoogleDrive = "GoogleDrive", - Grafana = "Grafana", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - Kafka = "Kafka", - KafkaConnect = "KafkaConnect", - Kinesis = "Kinesis", - Lightdash = "Lightdash", - Looker = "Looker", - MariaDB = "MariaDB", - Matillion = "Matillion", - Metabase = "Metabase", - MetadataES = "MetadataES", - MicroStrategy = "MicroStrategy", - Mlflow = "Mlflow", - Mode = "Mode", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Nifi = "Nifi", - OpenLineage = "OpenLineage", - OpenMetadata = "OpenMetadata", - OpenSearch = "OpenSearch", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - PowerBI = "PowerBI", - PowerBIReportServer = "PowerBIReportServer", - Presto = "Presto", - QlikCloud = "QlikCloud", - QlikSense = "QlikSense", - QuickSight = "QuickSight", - REST = "Rest", - Ranger = "Ranger", - Redash = "Redash", - Redpanda = "Redpanda", - Redshift = "Redshift", - S3 = "S3", - SAS = "SAS", - SQLite = "SQLite", - SageMaker = "SageMaker", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SharePoint = "SharePoint", - Sigma = "Sigma", - SingleStore = "SingleStore", - Sklearn = "Sklearn", - Snowflake = "Snowflake", - Snowplow = "Snowplow", - Spark = "Spark", - Spline = "Spline", - Ssas = "SSAS", - Ssis = "SSIS", - Stitch = "Stitch", - Superset = "Superset", - Synapse = "Synapse", - Tableau = "Tableau", - Teradata = "Teradata", - ThoughtSpot = "ThoughtSpot", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - VertexAI = "VertexAI", - Vertica = "Vertica", - Wherescape = "Wherescape", -} - -/** - * No manifest file available. Ingestion would look for bucket-level metadata file instead - * - * Storage Metadata Manifest file path config. - * - * Storage Metadata Manifest file HTTP path config. - * - * Storage Metadata Manifest file S3 path config. - * - * Storage Metadata Manifest file ADLS path config. - * - * Storage Metadata Manifest file GCS path config. - */ -export interface StorageMetadataConfigurationSource { - /** - * Storage Metadata manifest file path to extract locations to ingest from. - */ - manifestFilePath?: string; - /** - * Storage Metadata manifest http file path to extract locations to ingest from. - */ - manifestHttpPath?: string; - prefixConfig?: StorageMetadataBucketDetails; - securityConfig?: DbtSecurityConfigClass; -} - -/** - * Details of the bucket where the storage metadata manifest file is stored - */ -export interface StorageMetadataBucketDetails { - /** - * Name of the top level container where the storage metadata file is stored - */ - containerName: string; - /** - * Path of the folder where the storage metadata file is stored. If the file is at the root, - * you can keep it empty. - */ - objectPrefix?: string; -} - -/** - * Pipeline type - * - * Database Source Config Metadata Pipeline type - * - * Database Source Config Usage Pipeline type - * - * Dashboard Source Config Metadata Pipeline type - * - * Messaging Source Config Metadata Pipeline type - * - * Profiler Source Config Pipeline type - * - * Pipeline Source Config Metadata Pipeline type - * - * MlModel Source Config Metadata Pipeline type - * - * Object Store Source Config Metadata Pipeline type - * - * Drive Source Config Metadata Pipeline type - * - * Search Source Config Metadata Pipeline type - * - * DBT Config Pipeline type - * - * Pipeline Source Config For Application Pipeline type. Nothing is required. - * - * Api Source Config Metadata Pipeline type - * - * Reverse Ingestion Config Pipeline type - */ -export enum FluffyType { - APIMetadata = "ApiMetadata", - Application = "Application", - AutoClassification = "AutoClassification", - DashboardMetadata = "DashboardMetadata", - DataInsight = "dataInsight", - DatabaseLineage = "DatabaseLineage", - DatabaseMetadata = "DatabaseMetadata", - DatabaseUsage = "DatabaseUsage", - Dbt = "DBT", - DriveMetadata = "DriveMetadata", - MessagingMetadata = "MessagingMetadata", - MetadataToElasticSearch = "MetadataToElasticSearch", - MlModelMetadata = "MlModelMetadata", - PipelineMetadata = "PipelineMetadata", - Profiler = "Profiler", - ReverseIngestion = "ReverseIngestion", - SearchMetadata = "SearchMetadata", - StorageMetadata = "StorageMetadata", - TestSuite = "TestSuite", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts deleted file mode 100644 index bd0ffb3eedc..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts +++ /dev/null @@ -1,4168 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * TestSuite Pipeline Configuration. - */ -export interface TestSuitePipeline { - /** - * Fully qualified name of the entity to be tested, if we're working with a basic suite. - */ - entityFullyQualifiedName?: string; - /** - * Percentage of data or no. of rows we want to execute the profiler and tests on - */ - profileSample?: number; - profileSampleType?: ProfileSampleType; - samplingMethodType?: SamplingMethodType; - /** - * Service connections to be used for the logical test suite. - */ - serviceConnections?: ServiceConnections[]; - /** - * List of test cases to be executed on the entity. If null, all test cases will be executed. - */ - testCases?: string[]; - /** - * Pipeline type - */ - type: TestSuiteConfigType; -} - -/** - * Type of Profile Sample (percentage or rows) - */ -export enum ProfileSampleType { - Percentage = "PERCENTAGE", - Rows = "ROWS", -} - -/** - * Type of Sampling Method (BERNOULLI or SYSTEM) - */ -export enum SamplingMethodType { - Bernoulli = "BERNOULLI", - System = "SYSTEM", -} - -/** - * Service connections available for the logical test suite. - */ -export interface ServiceConnections { - /** - * Connection configuration for the source. ex: mysql , tableau connection. - */ - serviceConnection: ServiceConnection; - serviceName: string; -} - -/** - * Connection configuration for the source. ex: mysql , tableau connection. - * - * Supported services - * - * API Service Connection. - * - * Dashboard Connection. - * - * Database Connection. - * - * Metadata Service Connection. - * - * Pipeline Connection. - * - * MlModel Connection. - * - * Storage Connection. - * - * search Connection. - * - * Security Connection. - * - * Drive Connection. - */ -export interface ServiceConnection { - config?: ConfigObject; -} - -/** - * REST Connection Config - * - * Looker Connection Config - * - * Metabase Connection Config - * - * PowerBI Connection Config - * - * PowerBIReportServer Connection Config - * - * Redash Connection Config - * - * Superset Connection Config - * - * Tableau Connection Config - * - * Mode Connection Config - * - * Custom Dashboard Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Domo Dashboard Connection Config - * - * QuickSight Connection Config - * - * Qlik Sense Connection Config - * - * Lightdash Connection Config - * - * MicroStrategy Connection Config - * - * Qlik Cloud Connection Config - * - * Sigma Connection Config - * - * ThoughtSpot Connection Config - * - * Grafana Connection Config - * - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - * - * Kafka Connection Config - * - * Redpanda Connection Config - * - * Kinesis Connection Config - * - * Custom Messaging Service Connection to build a source that is not supported by - * OpenMetadata yet. - * - * Amundsen Connection Config - * - * Metadata to ElasticSearch Connection Config - * - * OpenMetadata Connection Config - * - * Atlas Connection Config - * - * Alation Connection Config - * - * Alation Sink Connection Config - * - * Airflow Metadata Database Connection Config - * - * Wherescape Metadata Database Connection Config - * - * SSIS Metadata Database Connection Config - * - * Glue Pipeline Connection Config - * - * Airbyte Metadata Database Connection Config - * - * Fivetran Metadata Database Connection Config - * - * Flink Metadata Connection Config - * - * Dagster Metadata Database Connection Config - * - * Nifi Metadata Pipeline Connection Config - * - * Domo Pipeline Connection Config - * - * Custom Pipeline Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Spline Metadata Database Connection Config - * - * Spark Metadata Pipeline Connection Config - * - * OpenLineage Connection Config - * - * KafkaConnect Connection Config - * - * DBTCloud Connection Config - * - * Matillion Connection - * - * Azure Data Factory Connection Config - * - * Stitch Connection - * - * Snowplow Pipeline Connection Config - * - * MlFlow Connection Config - * - * Sklearn Connection Config - * - * Custom MlModel Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * SageMaker Connection Config - * - * Google VertexAI Connection Config - * - * S3 Connection. - * - * ADLS Connection. - * - * GCS Connection. - * - * Custom Storage Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * ElasticSearch Connection. - * - * OpenSearch Connection Config - * - * Custom Search Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * Apache Ranger Connection Config - * - * Google Drive Connection Config - * - * SharePoint Connection Config - * - * Custom Drive Connection to build a source that is not supported. - */ -export interface ConfigObject { - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Documentation URL for the schema. - */ - docURL?: string; - /** - * Open API Schema URL. - */ - openAPISchemaURL?: string; - /** - * Supports Metadata Extraction. - */ - supportsMetadataExtraction?: boolean; - /** - * Generated Token to connect to OpenAPI Schema. - * - * token to connect to Qlik Cloud. - * - * Generated Token to connect to Databricks. - * - * To Connect to Dagster Cloud - * - * Generated Token to connect to DBTCloud. - * - * Token to connect to Stitch api doc - */ - token?: string; - /** - * REST API Type - * - * Service Type - * - * Custom dashboard service type - * - * Custom database service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * OpenSearch Type - * - * Custom search service type - */ - type?: ConfigType; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * User's Client ID. This user should have privileges to read all the metadata in Looker. - * - * client_id for PowerBI. - * - * Client ID for DOMO - * - * client_id for Sigma. - * - * Azure Application (client) ID for service principal authentication. - * - * Application (client) ID from Azure Active Directory - */ - clientId?: string; - /** - * User's Client Secret. - * - * clientSecret for PowerBI. - * - * clientSecret for Sigma. - * - * Azure Application client secret for service principal authentication. - * - * Application (client) secret from Azure Active Directory - */ - clientSecret?: string; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - */ - gitCredentials?: GitHubCredentials; - /** - * URL to the Looker instance. - * - * Host and Port of the Metabase instance. - * - * Dashboard URL for PowerBI service. - * - * Dashboard URL for PowerBI Report Server. - * - * URL for the Redash instance - * - * URL for the superset instance. - * - * Tableau Server url. - * - * URL for the mode instance. - * - * URL for the Qlik instance. - * - * Address for your running Lightdash instance - * - * Host and Port of the MicroStrategy instance. - * - * Host and Port of the Qlik Cloud instance. - * - * Sigma API url. - * - * ThoughtSpot instance URL. Example: https://my-company.thoughtspot.cloud - * - * URL to the Grafana instance. - * - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - * - * Host and port of the Amundsen Neo4j Connection. This expect a URI format like: - * bolt://localhost:7687. - * - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - * - * Host and port of the Atlas service. - * - * Host and port of the Alation service. - * - * Pipeline Service Management/UI URI. - * - * Pipeline Service Management/UI URL. - * - * Spline REST Server Host & Port. - * - * KafkaConnect Service Management/UI URI. - * - * Host and port of the Stitch API host - * - * Host and port of the ElasticSearch service. - * - * Host and port of the OpenSearch service. - * - * Apache Ranger Admin URL. - */ - hostPort?: string; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * API token to connect to Metabase. Use this instead of username/password for token-based - * authentication. - * - * API key of the redash instance to access. - * - * The personal access token you can generate in the Lightdash app under the user settings - * - * Service Account Token to authenticate to the Grafana APIs. Use Service Account Tokens - * (format: glsa_xxxx) for authentication. Legacy API Keys are no longer supported by - * Grafana as of January 2025. Both self-hosted and Grafana Cloud are supported. Requires - * Admin role for full metadata extraction. - * - * API key to authenticate with the SAP ERP APIs. - * - * Fivetran API Secret. - * - * API Key for Snowplow Console API - */ - apiKey?: string; - /** - * Password to connect to Metabase. Required for basic authentication. - * - * Password to connect to PowerBI report server. - * - * Password to connect to MicroStrategy. - * - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to the Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - * - * password to connect to the Amundsen Neo4j Connection. - * - * password to connect to the Atlas. - * - * Password to connect to Airbyte. - */ - password?: string; - /** - * Username to connect to Metabase. Required for basic authentication. - * - * Username to connect to PowerBI report server. - * - * Username for Redash - * - * Username to connect to MicroStrategy. This user should have privileges to read all the - * metadata in MicroStrategy. - * - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to the Salesforce. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - * - * username to connect to the Amundsen Neo4j Connection. - * - * username to connect to the Atlas. This user should have privileges to read all the - * metadata in Atlas. - * - * Username to connect to Airbyte. - */ - username?: string; - /** - * Authority URI for the PowerBI service. - */ - authorityURI?: string; - /** - * Display Table Name from source instead of renamed table name for datamodel tables - */ - displayTableNameFromSource?: boolean; - /** - * Entity Limit set here will be used to paginate the PowerBi APIs - */ - pagination_entity_per_page?: number; - /** - * Source to get the .pbit files to extract lineage information - */ - pbitFilesSource?: PowerBIPbitFilesSource; - /** - * PowerBI secrets. - */ - scope?: string[]; - /** - * Tenant ID for PowerBI. - * - * Azure Directory (tenant) ID for service principal authentication. - * - * Directory (tenant) ID from Azure Active Directory - */ - tenantId?: string; - /** - * Fetch the PowerBI metadata using admin APIs - */ - useAdminApis?: boolean; - /** - * Web Portal Virtual Directory Name. - */ - webPortalVirtualDirectory?: string; - /** - * Version of the Redash instance - */ - redashVersion?: string; - /** - * Choose between API or database connection fetch metadata from superset. - * - * Choose between Database connection or HDB User Store connection. - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Matillion Auth Configuration - */ - connection?: ConfigConnection; - /** - * Tableau API version. If not provided, the version will be used from the tableau server. - * - * Sigma API version. - * - * ThoughtSpot API version to use - * - * OpenMetadata server API version to use. - * - * Airbyte API version. - */ - apiVersion?: string; - /** - * Types of methods used to authenticate to the tableau instance - * - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - * - * Types of methods used to authenticate to the alation instance - * - * Authentication type to connect to Apache Ranger. - */ - authType?: AuthenticationTypeForTableau | NoConfigAuthenticationTypes; - /** - * Pagination limit used while querying the tableau metadata API for getting data sources - * - * Pagination limit used while querying the SAP ERP API for fetching the entities - * - * Pagination limit used for Alation APIs pagination - */ - paginationLimit?: number; - /** - * Proxy URL for the tableau server. If not provided, the hostPort will be used. This is - * used to generate the dashboard & Chart URL. - */ - proxyURL?: string; - /** - * Tableau Site Name. - * - * SharePoint site name - */ - siteName?: string; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; - /** - * Boolean marking if we need to verify the SSL certs for Grafana. Default to True. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Boolean marking if we need to verify the SSL certs for KafkaConnect REST API. True by - * default. - */ - verifySSL?: boolean | VerifySSL; - /** - * Access Token for Mode Dashboard - * - * Access token to connect to DOMO - */ - accessToken?: string; - /** - * Access Token Password for Mode Dashboard - */ - accessTokenPassword?: string; - /** - * Filter query parameter for some of the Mode API calls - */ - filterQueryParam?: string; - /** - * Mode Workspace Name - */ - workspaceName?: string; - connectionOptions?: { [key: string]: string }; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - * - * Secret token to connect to DOMO - */ - secretToken?: string; - /** - * AWS Account ID - */ - awsAccountId?: string; - awsConfig?: AWSCredentials; - /** - * The authentication method that the user uses to sign in. - */ - identityType?: IdentityType; - /** - * The Amazon QuickSight namespace that contains the dashboard IDs in this request ( To be - * provided when identityType is `ANONYMOUS` ) - */ - namespace?: string; - certificates?: QlikCertificatesBy; - /** - * Qlik Sense Base URL, used for genrating dashboard & chat url - */ - displayUrl?: string; - /** - * User Directory. - */ - userDirectory?: string; - /** - * User ID. - */ - userId?: string; - /** - * Validate Host Name - */ - validateHostName?: boolean; - /** - * The Project UUID for your Lightdash instance - */ - projectUUID?: string; - /** - * Use if your Lightdash instance is behind a proxy like (Cloud IAP) - */ - proxyAuthentication?: string; - /** - * The Space UUID for your Lightdash instance - */ - spaceUUID?: string; - /** - * Login Mode for Microstrategy's REST API connection. You can authenticate with one of the - * following authentication modes: `Standard (1)`, `Anonymous (8)`. Default will be - * `Standard (1)`. If you're using demo account for Microstrategy, it will be needed to - * authenticate through loginMode `8`. - */ - loginMode?: string; - /** - * MicroStrategy Project Name - * - * Project name to create the refreshToken. Can be anything - */ - projectName?: string; - /** - * Space types of Qlik Cloud to filter the dashboards ingested into the platform. - */ - spaceTypes?: SpaceType[]; - /** - * ThoughtSpot authentication configuration - */ - authentication?: Authenticationation; - /** - * Org ID for multi-tenant ThoughtSpot instances. This is applicable for ThoughtSpot Cloud - * only. - */ - orgId?: string; - /** - * Page size for pagination in API requests. Default is 100. - */ - pageSize?: number; - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - * - * Azure Credentials - * - * GCP Credentials for Google Drive API - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - * - * Connection timeout in seconds. - */ - connectionTimeout?: number | number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - * - * Available sources to fetch metadata. - */ - configSource?: DeltaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - */ - auth?: AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - * - * Snowplow BDP Organization ID - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - /** - * basic.auth.user.info schema registry config property, Client HTTP credentials in the form - * of username:password. - */ - basicAuthUserInfo?: string; - /** - * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - * - * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - */ - bootstrapServers?: string; - /** - * Confluent Kafka Consumer Config. From - * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - * - * Confluent Redpanda Consumer Config - */ - consumerConfig?: { [key: string]: any }; - /** - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - */ - consumerConfigSSL?: ConsumerConfigSSLClass; - /** - * sasl.mechanism Consumer Config property - */ - saslMechanism?: SaslMechanismType; - /** - * sasl.password consumer config property - */ - saslPassword?: string; - /** - * sasl.username consumer config property - */ - saslUsername?: string; - /** - * Confluent Kafka Schema Registry Config. From - * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient - * - * Confluent Redpanda Schema Registry Config. - */ - schemaRegistryConfig?: { [key: string]: any }; - /** - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ - schemaRegistrySSL?: ConsumerConfigSSLClass; - /** - * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get - * topic schema from registry. - */ - schemaRegistryTopicSuffixName?: string; - /** - * Confluent Kafka Schema Registry URL. - * - * Confluent Redpanda Schema Registry URL. - */ - schemaRegistryURL?: string; - /** - * security.protocol consumer config property - * - * Kafka security protocol config - */ - securityProtocol?: KafkaSecurityProtocol; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Enable encryption for the Amundsen Neo4j Connection. - */ - encrypted?: boolean; - /** - * Maximum connection lifetime for the Amundsen Neo4j Connection. - */ - maxConnectionLifeTime?: number; - /** - * Enable SSL validation for the Amundsen Neo4j Connection. - */ - validateSSL?: boolean; - /** - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - /** - * List of entities that you need to reindex - */ - entities?: string[]; - recreateIndex?: boolean; - runMode?: RunMode; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * service type of the data source. - */ - databaseServiceName?: string[]; - /** - * Name of the Entity Type available in Atlas. - */ - entity_type?: string; - /** - * service type of the messaging source - * - * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. - * e.g. local_kafka - */ - messagingServiceName?: string[] | string; - /** - * Custom OpenMetadata Classification name for alation tags. - */ - alationTagClassificationName?: string; - /** - * Specifies if hidden datasources should be included while ingesting. - */ - includeHiddenDatasources?: boolean; - /** - * Specifies if undeployed datasources should be included while ingesting. - */ - includeUndeployedDatasources?: boolean; - /** - * Specifies if Dashboards are to be ingested while running the ingestion job. - */ - ingestDashboards?: boolean; - /** - * Specifies if Datasources are to be ingested while running the ingestion job. - */ - ingestDatasources?: boolean; - /** - * Specifies if Domains are to be ingested while running the ingestion job. - */ - ingestDomains?: boolean; - /** - * Specifies if Knowledge Articles are to be ingested while running the ingestion job. - */ - ingestKnowledgeArticles?: boolean; - /** - * Specifies if Users and Groups are to be ingested while running the ingestion job. - */ - ingestUsersAndGroups?: boolean; - datasourceLinks?: { [key: string]: string }; - /** - * Pipeline Service Number Of Status - */ - numberOfStatus?: number; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Underlying database connection - */ - databaseConnection?: DatabaseConnectionClass; - /** - * Underlying storage connection - */ - packageConnection?: S3Connection | string; - /** - * Fivetran API Secret. - */ - apiSecret?: string; - /** - * Fivetran API Limit For Pagination. - */ - limit?: number; - /** - * URL to the Dagster instance - * - * DBT cloud Access URL. - */ - host?: string; - /** - * Connection Time Limit Between OM and Dagster Graphql API in second - */ - timeout?: number; - /** - * We support username/password or client certificate authentication - */ - nifiConfig?: NifiCredentialsConfiguration; - /** - * Spline UI Host & Port. - */ - uiHostPort?: string; - /** - * service type of the messaging source - */ - brokersUrl?: string; - /** - * consumer group name - */ - consumerGroupName?: string; - /** - * initial Kafka consumer offset - */ - consumerOffsets?: InitialConsumerOffsets; - /** - * max allowed wait time - */ - poolTimeout?: number; - /** - * SASL Configuration details. - */ - saslConfig?: SASLClientConfig; - /** - * max allowed inactivity time - */ - sessionTimeout?: number; - /** - * topic from where Open lineage events will be pulled - */ - topicName?: string; - /** - * We support username/password or No Authentication - */ - KafkaConnectConfig?: UsernamePasswordAuthentication; - /** - * ID of your DBT cloud account - */ - accountId?: string; - /** - * DBT cloud Metadata API URL. - */ - discoveryAPI?: string; - /** - * List of IDs of your DBT cloud jobs seperated by comma `,` - */ - jobIds?: string[]; - /** - * Number of runs to fetch from DBT cloud - */ - numberOfRuns?: number; - /** - * List of IDs of your DBT cloud projects seperated by comma `,` - */ - projectIds?: string[]; - /** - * The name of your azure data factory. - */ - factory_name?: string; - /** - * The name of your resource group the data factory is associated with. - */ - resource_group_name?: string; - /** - * Number of days in the past to filter pipeline runs. - */ - run_filter_days?: number; - /** - * The azure subscription identifier. - */ - subscription_id?: string; - /** - * Cloud provider where Snowplow is deployed - */ - cloudProvider?: CloudProvider; - /** - * Path to pipeline configuration files for Community deployment - */ - configPath?: string; - /** - * Snowplow Console URL for BDP deployment - */ - consoleUrl?: string; - /** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - */ - deployment?: SnowplowDeployment; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Mlflow Model registry backend. E.g., - * mysql+pymysql://mlflow:password@localhost:3307/experiments - */ - registryUri?: string; - /** - * Mlflow Experiment tracking URI. E.g., http://localhost:5000 - */ - trackingUri?: string; - /** - * location/region of google cloud project - */ - location?: string; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Connection Timeout in Seconds - */ - connectionTimeoutSecs?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Email to impersonate using domain-wide delegation - */ - delegatedEmail?: string; - /** - * Specific shared drive ID to connect to - * - * SharePoint drive ID. If not provided, default document library will be used - */ - driveId?: string; - /** - * Extract metadata only for Google Sheets files - */ - includeGoogleSheets?: boolean; - /** - * Include shared/team drives in metadata extraction - */ - includeTeamDrives?: boolean; - /** - * SharePoint site URL - */ - siteUrl?: string; - [property: string]: any; -} - -/** - * We support username/password or No Authentication - * - * username/password auth - */ -export interface UsernamePasswordAuthentication { - /** - * KafkaConnect password to authenticate to the API. - */ - password?: string; - /** - * KafkaConnect user to authenticate to the API. - */ - username?: string; -} - -/** - * Regex to only fetch api collections with names matching the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex exclude or include charts that matches the pattern. - * - * Regex to exclude or include dashboards that matches the pattern. - * - * Regex exclude or include data models that matches the pattern. - * - * Regex to exclude or include projects that matches the pattern. - * - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - * - * Regex to only fetch topics that matches the pattern. - * - * Regex exclude pipelines. - * - * Regex to only fetch containers that matches the pattern. - * - * Regex to only fetch MlModels with names matching the pattern. - * - * Regex to only fetch search indexes that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Types of methods used to authenticate to the tableau instance - * - * Basic Auth Credentials - * - * Access Token Auth Credentials - * - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - * - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * API Access Token Auth Credentials - * - * Basic Auth Configuration for ElasticSearch - * - * SSL Certificates By Path - * - * AWS credentials configs. - * - * Authentication type to connect to Apache Ranger. - * - * Configuration for connecting to Ranger Basic Auth. - */ -export interface AuthenticationTypeForTableau { - /** - * Password to access the service. - * - * Password to connect to source. - * - * Elastic Search Password for Login - * - * Ranger password to authenticate to the API. - */ - password?: string; - /** - * Username to access the service. - * - * Elastic Search Username for Login - * - * Ranger user to authenticate to the API. - */ - username?: string; - /** - * Personal Access Token Name. - */ - personalAccessTokenName?: string; - /** - * Personal Access Token Secret. - */ - personalAccessTokenSecret?: string; - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; - /** - * Access Token for the API - */ - accessToken?: string; - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -/** - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * Basic Auth Credentials - * - * API Access Token Auth Credentials - */ -export interface Authenticationation { - /** - * Password to access the service. - */ - password?: string; - /** - * Username to access the service. - */ - username?: string; - /** - * Access Token for the API - */ - accessToken?: string; -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: Authentication; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum Authentication { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: Credentials | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Qlik Authentication Certificate By Values - * - * Qlik Authentication Certificate File Path - */ -export interface QlikCertificatesBy { - sslConfig?: ConsumerConfigSSLClass; - /** - * Client Certificate - */ - clientCertificate?: string; - /** - * Client Key Certificate. - */ - clientKeyCertificate?: string; - /** - * Root Certificate. - */ - rootCertificate?: string; - [property: string]: any; -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface ConsumerConfigSSLClass { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Cloud provider where Snowplow is deployed - */ -export enum CloudProvider { - Aws = "AWS", - Azure = "Azure", - Gcp = "GCP", -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface DeltaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConfigSourceConnection; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: SecurityConfigClass; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConfigSourceConnection { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - * - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * AWS credentials configs. - */ -export interface SecurityConfigClass { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Choose between API or database connection fetch metadata from superset. - * - * Superset API Connection Config - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - * - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Lineage Backend Connection Config - * - * SQLite Database Connection Config - * - * Matillion Auth Configuration - * - * Matillion ETL Auth Config. - */ -export interface ConfigConnection { - /** - * Password for Superset. - * - * Password to connect to Hana. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to the Matillion. - */ - password?: string; - /** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ - provider?: Provider; - /** - * SSL Configuration details. - */ - sslConfig?: ConnectionSSLConfig; - /** - * Username for Superset. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to Hana. This user should have privileges to read all the metadata. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to the Matillion. This user should have privileges to read all the - * metadata in Matillion. - */ - username?: string; - verifySSL?: VerifySSL; - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - * - * Host and port of the Hana service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Matillion Host - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: ConnectionScheme; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: ConnectionType; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - supportsViewLineageExtraction?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ -export enum Provider { - DB = "db", - LDAP = "ldap", -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum ConnectionScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", - SqlitePysqlite = "sqlite+pysqlite", -} - -/** - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - */ -export interface ConnectionSSLConfig { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum ConnectionType { - Backend = "Backend", - MatillionETL = "MatillionETL", - Mysql = "Mysql", - Postgres = "Postgres", - SQLite = "SQLite", -} - -/** - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * initial Kafka consumer offset - */ -export enum InitialConsumerOffsets { - Earliest = "earliest", - Latest = "latest", -} - -/** - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Underlying database connection - * - * Mssql Database Connection Config - */ -export interface DatabaseConnectionClass { - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Host and port of the MSSQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to MSSQL. - */ - password?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: MssqlScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: MssqlType; - /** - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - */ - username?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum MssqlScheme { - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", -} - -/** - * Service Type - * - * Service type. - */ -export enum MssqlType { - Mssql = "Mssql", -} - -/** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - * - * Snowplow deployment type - */ -export enum SnowplowDeployment { - Bdp = "BDP", - Community = "Community", -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - * - * Do not set any credentials. Note that credentials are required to extract .lkml views and - * their lineage. - * - * Credentials for a GitHub repository - * - * Credentials for a BitBucket repository - * - * Credentials for a Gitlab repository - */ -export interface GitHubCredentials { - repositoryName?: string; - repositoryOwner?: string; - token?: string; - /** - * Credentials Type - */ - type?: GitHubCredentialsType; - /** - * Main production branch of the repository. E.g., `main` - */ - branch?: string; -} - -/** - * Credentials Type - * - * GitHub Credentials type - * - * BitBucket Credentials type - * - * Gitlab Credentials type - */ -export enum GitHubCredentialsType { - BitBucket = "BitBucket", - GitHub = "GitHub", - Gitlab = "Gitlab", -} - -/** - * The authentication method that the user uses to sign in. - */ -export enum IdentityType { - Anonymous = "ANONYMOUS", - Iam = "IAM", - Quicksight = "QUICKSIGHT", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: ConsumerConfigSSLClass; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * We support username/password or client certificate authentication - * - * Configuration for connecting to Nifi Basic Auth. - * - * Configuration for connecting to Nifi Client Certificate Auth. - */ -export interface NifiCredentialsConfiguration { - /** - * Nifi password to authenticate to the API. - */ - password?: string; - /** - * Nifi user to authenticate to the API. - */ - username?: string; - /** - * Boolean marking if we need to verify the SSL certs for Nifi. False by default. - */ - verifySSL?: boolean; - /** - * Path to the root CA certificate - */ - certificateAuthorityPath?: string; - /** - * Path to the client certificate - */ - clientCertificatePath?: string; - /** - * Path to the client key - */ - clientkeyPath?: string; -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * S3 Connection. - */ -export interface S3Connection { - awsConfig: AWSCredentials; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: S3Type; -} - -/** - * Service Type - * - * S3 service type - */ -export enum S3Type { - S3 = "S3", -} - -/** - * Source to get the .pbit files to extract lineage information - * - * Local config source where no extra information needs to be sent. - * - * Azure storage config for pbit files - * - * GCS storage config for pbit files - * - * S3 storage config for pbit files - */ -export interface PowerBIPbitFilesSource { - /** - * Directory path for the pbit files - */ - path?: string; - /** - * pbit File Configuration type - */ - pbitFileConfigType?: PbitFileConfigType; - /** - * Path of the folder where the .pbit files will be unzipped and datamodel schema will be - * extracted - */ - pbitFilesExtractDir?: string; - prefixConfig?: BucketDetails; - securityConfig?: SecurityConfigClass; -} - -/** - * pbit File Configuration type - */ -export enum PbitFileConfigType { - Azure = "azure", - Gcs = "gcs", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the .pbit files are stored - */ -export interface BucketDetails { - /** - * Name of the bucket where the .pbit files are stored - */ - bucketName?: string; - /** - * Path of the folder where the .pbit files are stored - */ - objectPrefix?: string; -} - -/** - * This schema publisher run modes. - */ -export enum RunMode { - Batch = "batch", - Stream = "stream", -} - -/** - * SASL Configuration details. - * - * SASL client configuration. - */ -export interface SASLClientConfig { - /** - * SASL security mechanism - */ - saslMechanism?: SaslMechanismType; - /** - * The SASL authentication password. - */ - saslPassword?: string; - /** - * The SASL authentication username. - */ - saslUsername?: string; -} - -/** - * sasl.mechanism Consumer Config property - * - * SASL Mechanism consumer config property - * - * SASL security mechanism - */ -export enum SaslMechanismType { - Gssapi = "GSSAPI", - Oauthbearer = "OAUTHBEARER", - Plain = "PLAIN", - ScramSHA256 = "SCRAM-SHA-256", - ScramSHA512 = "SCRAM-SHA-512", -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * security.protocol consumer config property - * - * Kafka security protocol config - */ -export enum KafkaSecurityProtocol { - Plaintext = "PLAINTEXT", - SSL = "SSL", - SaslPlaintext = "SASL_PLAINTEXT", - SaslSSL = "SASL_SSL", -} - -export enum SpaceType { - Data = "Data", - Managed = "Managed", - Personal = "Personal", - Shared = "Shared", -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Config - */ -export interface SSLConfigObject { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; - /** - * SSL Certificates - */ - certificates?: SSLCertificates; - [property: string]: any; -} - -/** - * SSL Certificates - * - * SSL Configuration details. - * - * SSL Certificates By Path - * - * SSL Certificates By Values - */ -export interface SSLCertificates { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * CA Certificate Value - */ - caCertValue?: string; - /** - * Client Certificate Value - */ - clientCertValue?: string; - /** - * Private Key Value - */ - privateKeyValue?: string; - /** - * Staging Directory Path - */ - stagingDir?: string; -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * REST API Type - * - * REST API type - * - * Service Type - * - * Looker service type - * - * Metabase service type - * - * PowerBI service type - * - * PowerBIReportServer service type - * - * Redash service type - * - * Superset service type - * - * Tableau service type - * - * Mode service type - * - * Custom dashboard service type - * - * service type - * - * QuickSight service type - * - * Qlik sense service type - * - * Lightdash service type - * - * MicroStrategy service type - * - * Qlik Cloud service type - * - * Sigma service type - * - * ThoughtSpot service type - * - * Grafana service type - * - * Service type. - * - * Custom database service type - * - * Kafka service type - * - * Redpanda service type - * - * Custom messaging service type - * - * Amundsen service type - * - * Metadata to Elastic Search type - * - * OpenMetadata service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * S3 service type - * - * ADLS service type - * - * Gcs service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * ElasticSearch service type - * - * OpenSearch Type - * - * OpenSearch service type - * - * Custom search service type - * - * Apache Ranger service type - * - * Google Drive service type - * - * SharePoint service type - * - * Custom Drive service type - */ -export enum ConfigType { - Adls = "ADLS", - Airbyte = "Airbyte", - Airflow = "Airflow", - Alation = "Alation", - AlationSink = "AlationSink", - Amundsen = "Amundsen", - Athena = "Athena", - Atlas = "Atlas", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDashboard = "CustomDashboard", - CustomDatabase = "CustomDatabase", - CustomDrive = "CustomDrive", - CustomMessaging = "CustomMessaging", - CustomMlModel = "CustomMlModel", - CustomPipeline = "CustomPipeline", - CustomSearch = "CustomSearch", - CustomStorage = "CustomStorage", - DBTCloud = "DBTCloud", - Dagster = "Dagster", - DataFactory = "DataFactory", - Databricks = "Databricks", - DatabricksPipeline = "DatabricksPipeline", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDashboard = "DomoDashboard", - DomoDatabase = "DomoDatabase", - DomoPipeline = "DomoPipeline", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - ElasticSearch = "ElasticSearch", - Epic = "Epic", - Exasol = "Exasol", - Fivetran = "Fivetran", - Flink = "Flink", - Gcs = "GCS", - Glue = "Glue", - GluePipeline = "GluePipeline", - GoogleDrive = "GoogleDrive", - Grafana = "Grafana", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - Kafka = "Kafka", - KafkaConnect = "KafkaConnect", - Kinesis = "Kinesis", - Lightdash = "Lightdash", - Looker = "Looker", - MariaDB = "MariaDB", - Matillion = "Matillion", - Metabase = "Metabase", - MetadataES = "MetadataES", - MicroStrategy = "MicroStrategy", - Mlflow = "Mlflow", - Mode = "Mode", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Nifi = "Nifi", - OpenLineage = "OpenLineage", - OpenMetadata = "OpenMetadata", - OpenSearch = "OpenSearch", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - PowerBI = "PowerBI", - PowerBIReportServer = "PowerBIReportServer", - Presto = "Presto", - QlikCloud = "QlikCloud", - QlikSense = "QlikSense", - QuickSight = "QuickSight", - REST = "Rest", - Ranger = "Ranger", - Redash = "Redash", - Redpanda = "Redpanda", - Redshift = "Redshift", - S3 = "S3", - SAS = "SAS", - SQLite = "SQLite", - SageMaker = "SageMaker", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SharePoint = "SharePoint", - Sigma = "Sigma", - SingleStore = "SingleStore", - Sklearn = "Sklearn", - Snowflake = "Snowflake", - Snowplow = "Snowplow", - Spark = "Spark", - Spline = "Spline", - Ssas = "SSAS", - Ssis = "SSIS", - Stitch = "Stitch", - Superset = "Superset", - Synapse = "Synapse", - Tableau = "Tableau", - Teradata = "Teradata", - ThoughtSpot = "ThoughtSpot", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - VertexAI = "VertexAI", - Vertica = "Vertica", - Wherescape = "Wherescape", -} - -/** - * Pipeline type - * - * Pipeline Source Config Metadata Pipeline type - */ -export enum TestSuiteConfigType { - TestSuite = "TestSuite", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts deleted file mode 100644 index 6334f00486d..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts +++ /dev/null @@ -1,6289 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * OpenMetadata Ingestion Framework definition. - */ -export interface Workflow { - /** - * Unique identifier that identifies this pipeline. - */ - id?: string; - /** - * Name that identifies this pipeline instance uniquely. - */ - name: string; - /** - * OpenMetadata Ingestion Workflow Config. - */ - openMetadataWorkflowConfig: OpenMetadataWorkflowConfig; -} - -/** - * OpenMetadata Ingestion Workflow Config. - */ -export interface OpenMetadataWorkflowConfig { - bulkSink?: BulkSink; - /** - * Fully qualified name of ingestion pipeline, used to identify the current ingestion - * pipeline - */ - ingestionPipelineFQN?: string; - /** - * Unique identifier of pipeline run, used to identify the current pipeline run - */ - pipelineRunId?: string; - processor?: Processor; - sink?: Sink; - source: Source; - stage?: Stage; - workflowConfig: WorkflowConfig; -} - -/** - * Configuration for BulkSink Component in the OpenMetadata Ingestion Framework. - */ -export interface BulkSink { - config?: { [key: string]: any }; - /** - * Type of BulkSink component ex: metadata-usage - */ - type: string; -} - -/** - * Configuration for Processor Component in the OpenMetadata Ingestion Framework. - */ -export interface Processor { - config?: { [key: string]: any }; - /** - * Type of processor component ex: pii-processor - */ - type: string; -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface Sink { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * Configuration for Source component in OpenMetadata Ingestion Framework. - */ -export interface Source { - /** - * Connection configuration for the source. ex: mysql , tableau connection. - */ - serviceConnection?: ServiceConnection; - /** - * Type of the source connector ex: mysql, snowflake, tableau etc.. - */ - serviceName?: string; - sourceConfig: SourceConfig; - /** - * Type of the source connector ex: mysql, snowflake, tableau etc.. - */ - type: string; -} - -/** - * Connection configuration for the source. ex: mysql , tableau connection. - * - * Supported services - * - * API Service Connection. - * - * Dashboard Connection. - * - * Database Connection. - * - * Metadata Service Connection. - * - * Pipeline Connection. - * - * MlModel Connection. - * - * Storage Connection. - * - * search Connection. - * - * Security Connection. - * - * Drive Connection. - */ -export interface ServiceConnection { - config?: ConfigObject; -} - -/** - * REST Connection Config - * - * Looker Connection Config - * - * Metabase Connection Config - * - * PowerBI Connection Config - * - * PowerBIReportServer Connection Config - * - * Redash Connection Config - * - * Superset Connection Config - * - * Tableau Connection Config - * - * Mode Connection Config - * - * Custom Dashboard Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Domo Dashboard Connection Config - * - * QuickSight Connection Config - * - * Qlik Sense Connection Config - * - * Lightdash Connection Config - * - * MicroStrategy Connection Config - * - * Qlik Cloud Connection Config - * - * Sigma Connection Config - * - * ThoughtSpot Connection Config - * - * Grafana Connection Config - * - * Google BigQuery Connection Config - * - * Google BigTable Connection Config - * - * AWS Athena Connection Config - * - * Azure SQL Connection Config - * - * Clickhouse Connection Config - * - * Databricks Connection Config - * - * Db2 Connection Config - * - * DeltaLake Database Connection Config - * - * Druid Connection Config - * - * DynamoDB Connection Config - * - * Glue Connection Config - * - * Hive SQL Connection Config - * - * Impala SQL Connection Config - * - * MariaDB Database Connection Config - * - * Mssql Database Connection Config - * - * Mysql Database Connection Config - * - * SQLite Database Connection Config - * - * Oracle Database Connection Config - * - * Postgres Database Connection Config - * - * Presto Database Connection Config - * - * Redshift Connection Config - * - * Salesforce Connection Config - * - * SingleStore Database Connection Config - * - * Snowflake Connection Config - * - * Trino Connection Config - * - * Vertica Connection Config - * - * PinotDB Database Connection Config - * - * Datalake Connection Config - * - * Domo Database Connection Config - * - * Custom Database Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Sap Hana Database Connection Config - * - * MongoDB Connection Config - * - * Cassandra Connection Config - * - * Couchbase Connection Config - * - * Greenplum Database Connection Config - * - * Doris Database Connection Config - * - * UnityCatalog Connection Config - * - * SAS Connection Config - * - * Iceberg Catalog Connection Config - * - * Teradata Database Connection Config - * - * Sap ERP Database Connection Config - * - * Synapse Database Connection Config - * - * Exasol Database Connection Config - * - * Cockroach Database Connection Config - * - * SSAS Metadata Database Connection Config - * - * Epic FHIR Connection Config - * - * ServiceNow Connection Config - * - * Kafka Connection Config - * - * Redpanda Connection Config - * - * Kinesis Connection Config - * - * Custom Messaging Service Connection to build a source that is not supported by - * OpenMetadata yet. - * - * Amundsen Connection Config - * - * Metadata to ElasticSearch Connection Config - * - * OpenMetadata Connection Config - * - * Atlas Connection Config - * - * Alation Connection Config - * - * Alation Sink Connection Config - * - * Airflow Metadata Database Connection Config - * - * Wherescape Metadata Database Connection Config - * - * SSIS Metadata Database Connection Config - * - * Glue Pipeline Connection Config - * - * Airbyte Metadata Database Connection Config - * - * Fivetran Metadata Database Connection Config - * - * Flink Metadata Connection Config - * - * Dagster Metadata Database Connection Config - * - * Nifi Metadata Pipeline Connection Config - * - * Domo Pipeline Connection Config - * - * Custom Pipeline Service connection to build a source that is not supported by - * OpenMetadata yet. - * - * Spline Metadata Database Connection Config - * - * Spark Metadata Pipeline Connection Config - * - * OpenLineage Connection Config - * - * KafkaConnect Connection Config - * - * DBTCloud Connection Config - * - * Matillion Connection - * - * Azure Data Factory Connection Config - * - * Stitch Connection - * - * Snowplow Pipeline Connection Config - * - * MlFlow Connection Config - * - * Sklearn Connection Config - * - * Custom MlModel Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * SageMaker Connection Config - * - * Google VertexAI Connection Config - * - * S3 Connection. - * - * ADLS Connection. - * - * GCS Connection. - * - * Custom Storage Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * ElasticSearch Connection. - * - * OpenSearch Connection Config - * - * Custom Search Service connection to build a source that is not supported by OpenMetadata - * yet. - * - * Apache Ranger Connection Config - * - * Google Drive Connection Config - * - * SharePoint Connection Config - * - * Custom Drive Connection to build a source that is not supported. - */ -export interface ConfigObject { - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Documentation URL for the schema. - */ - docURL?: string; - /** - * Open API Schema URL. - */ - openAPISchemaURL?: string; - /** - * Supports Metadata Extraction. - */ - supportsMetadataExtraction?: boolean; - /** - * Generated Token to connect to OpenAPI Schema. - * - * token to connect to Qlik Cloud. - * - * Generated Token to connect to Databricks. - * - * To Connect to Dagster Cloud - * - * Generated Token to connect to DBTCloud. - * - * Token to connect to Stitch api doc - */ - token?: string; - /** - * REST API Type - * - * Service Type - * - * Custom dashboard service type - * - * Custom database service type - * - * Custom messaging service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * OpenSearch Type - * - * Custom search service type - */ - type?: PurpleType; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * User's Client ID. This user should have privileges to read all the metadata in Looker. - * - * client_id for PowerBI. - * - * Client ID for DOMO - * - * client_id for Sigma. - * - * Azure Application (client) ID for service principal authentication. - * - * Application (client) ID from Azure Active Directory - */ - clientId?: string; - /** - * User's Client Secret. - * - * clientSecret for PowerBI. - * - * clientSecret for Sigma. - * - * Azure Application client secret for service principal authentication. - * - * Application (client) secret from Azure Active Directory - */ - clientSecret?: string; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - */ - gitCredentials?: GitHubCredentials; - /** - * URL to the Looker instance. - * - * Host and Port of the Metabase instance. - * - * Dashboard URL for PowerBI service. - * - * Dashboard URL for PowerBI Report Server. - * - * URL for the Redash instance - * - * URL for the superset instance. - * - * Tableau Server url. - * - * URL for the mode instance. - * - * URL for the Qlik instance. - * - * Address for your running Lightdash instance - * - * Host and Port of the MicroStrategy instance. - * - * Host and Port of the Qlik Cloud instance. - * - * Sigma API url. - * - * ThoughtSpot instance URL. Example: https://my-company.thoughtspot.cloud - * - * URL to the Grafana instance. - * - * BigQuery APIs URL. - * - * Host and port of the AzureSQL service. - * - * Host and port of the Clickhouse service. - * - * Host and port of the Databricks service. - * - * Host and port of the DB2 service. - * - * Host and port of the Druid service. - * - * Host and port of the Hive service. - * - * Host and port of the Impala service. - * - * Host and port of the MariaDB service. - * - * Host and port of the MSSQL service. - * - * Host and port of the MySQL service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Host and port of the Oracle service. - * - * Host and port of the source service. - * - * Host and port of the Presto service. - * - * Host and port of the Redshift service. - * - * Host and port of the SingleStore service. - * - * Host and port of the Trino service. - * - * Host and port of the Vertica service. - * - * Host and port of the PinotDB Broker service. - * - * Host and port of the MongoDB service when using the `mongodb` connection scheme. Only - * host when using the `mongodb+srv` scheme. - * - * Host and port of the Cassandra service when using the `cassandra` connection scheme. Only - * host when using the `cassandra+srv` scheme. - * - * Host and port of the Doris service. - * - * Host and port of the Teradata service. - * - * Host and Port of the SAP ERP instance. - * - * Host and port of the Azure Synapse service. - * - * Host and port of the Cockrooach service. - * - * ServiceNow instance URL (e.g., https://your-instance.service-now.com) - * - * Host and port of the Amundsen Neo4j Connection. This expect a URI format like: - * bolt://localhost:7687. - * - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - * - * Host and port of the Atlas service. - * - * Host and port of the Alation service. - * - * Pipeline Service Management/UI URI. - * - * Pipeline Service Management/UI URL. - * - * Spline REST Server Host & Port. - * - * KafkaConnect Service Management/UI URI. - * - * Host and port of the Stitch API host - * - * Host and port of the ElasticSearch service. - * - * Host and port of the OpenSearch service. - * - * Apache Ranger Admin URL. - */ - hostPort?: string; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * API token to connect to Metabase. Use this instead of username/password for token-based - * authentication. - * - * API key of the redash instance to access. - * - * The personal access token you can generate in the Lightdash app under the user settings - * - * Service Account Token to authenticate to the Grafana APIs. Use Service Account Tokens - * (format: glsa_xxxx) for authentication. Legacy API Keys are no longer supported by - * Grafana as of January 2025. Both self-hosted and Grafana Cloud are supported. Requires - * Admin role for full metadata extraction. - * - * API key to authenticate with the SAP ERP APIs. - * - * Fivetran API Secret. - * - * API Key for Snowplow Console API - */ - apiKey?: string; - /** - * Password to connect to Metabase. Required for basic authentication. - * - * Password to connect to PowerBI report server. - * - * Password to connect to MicroStrategy. - * - * Password to connect to AzureSQL. - * - * Password to connect to Clickhouse. - * - * Password to connect to DB2. - * - * Password to connect to Druid. - * - * Password to connect to Hive. - * - * Password to connect to Impala. - * - * Password to connect to MariaDB. - * - * Password to connect to MSSQL. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to Oracle. - * - * Password to connect to Presto. - * - * Password to connect to Redshift. - * - * Password to connect to the Salesforce. - * - * Password to connect to SingleStore. - * - * Password to connect to Snowflake. - * - * Password to connect to Vertica. - * - * password to connect to the PinotDB. - * - * Password to connect to MongoDB. - * - * Password to connect to Couchbase. - * - * Password to connect to Doris. - * - * Password to connect to SAS Viya - * - * Password to connect to Teradata. - * - * Password to connect to Azure Synapse. - * - * Password to connect to Exasol. - * - * Password - * - * Password to connect to ServiceNow. - * - * password to connect to the Amundsen Neo4j Connection. - * - * password to connect to the Atlas. - * - * Password to connect to Airbyte. - */ - password?: string; - /** - * Username to connect to Metabase. Required for basic authentication. - * - * Username to connect to PowerBI report server. - * - * Username for Redash - * - * Username to connect to MicroStrategy. This user should have privileges to read all the - * metadata in MicroStrategy. - * - * Username to connect to AzureSQL. This user should have privileges to read the metadata. - * - * Username to connect to Clickhouse. This user should have privileges to read all the - * metadata in Clickhouse. - * - * Username to connect to DB2. This user should have privileges to read all the metadata in - * DB2. - * - * Username to connect to Druid. This user should have privileges to read all the metadata - * in Druid. - * - * Username to connect to Hive. This user should have privileges to read all the metadata in - * Hive. - * - * Username to connect to Impala. This user should have privileges to read all the metadata - * in Impala. - * - * Username to connect to MariaDB. This user should have privileges to read all the metadata - * in MariaDB. - * - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to Oracle. This user should have privileges to read all the metadata - * in Oracle. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to Presto. This user should have privileges to read all the metadata - * in Postgres. - * - * Username to connect to Redshift. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to the Salesforce. This user should have privileges to read all the - * metadata in Redshift. - * - * Username to connect to SingleStore. This user should have privileges to read all the - * metadata in MySQL. - * - * Username to connect to Snowflake. This user should have privileges to read all the - * metadata in Snowflake. - * - * Username to connect to Trino. This user should have privileges to read all the metadata - * in Trino. - * - * Username to connect to Vertica. This user should have privileges to read all the metadata - * in Vertica. - * - * username to connect to the PinotDB. This user should have privileges to read all the - * metadata in PinotDB. - * - * Username to connect to MongoDB. This user should have privileges to read all the metadata - * in MongoDB. - * - * Username to connect to Cassandra. This user should have privileges to read all the - * metadata in Cassandra. - * - * Username to connect to Couchbase. This user should have privileges to read all the - * metadata in Couchbase. - * - * Username to connect to Greenplum. This user should have privileges to read all the - * metadata in Greenplum. - * - * Username to connect to Doris. This user should have privileges to read all the metadata - * in Doris. - * - * Username to connect to SAS Viya. - * - * Username to connect to Teradata. This user should have privileges to read all the - * metadata in Teradata. - * - * Username to connect to Azure Synapse. This user should have privileges to read all the - * metadata in Azure Synapse. - * - * Username to connect to Exasol. This user should have privileges to read all the metadata - * in Exasol. - * - * Username to connect to Cockroach. This user should have privileges to read all the - * metadata in Cockroach. - * - * Username - * - * Username to connect to ServiceNow. This user should have read access to sys_db_object and - * sys_dictionary tables. - * - * username to connect to the Amundsen Neo4j Connection. - * - * username to connect to the Atlas. This user should have privileges to read all the - * metadata in Atlas. - * - * Username to connect to Airbyte. - */ - username?: string; - /** - * Authority URI for the PowerBI service. - */ - authorityURI?: string; - /** - * Display Table Name from source instead of renamed table name for datamodel tables - */ - displayTableNameFromSource?: boolean; - /** - * Entity Limit set here will be used to paginate the PowerBi APIs - */ - pagination_entity_per_page?: number; - /** - * Source to get the .pbit files to extract lineage information - */ - pbitFilesSource?: PowerBIPbitFilesSource; - /** - * PowerBI secrets. - */ - scope?: string[]; - /** - * Tenant ID for PowerBI. - * - * Azure Directory (tenant) ID for service principal authentication. - * - * Directory (tenant) ID from Azure Active Directory - */ - tenantId?: string; - /** - * Fetch the PowerBI metadata using admin APIs - */ - useAdminApis?: boolean; - /** - * Web Portal Virtual Directory Name. - */ - webPortalVirtualDirectory?: string; - /** - * Version of the Redash instance - */ - redashVersion?: string; - /** - * Choose between API or database connection fetch metadata from superset. - * - * Choose between Database connection or HDB User Store connection. - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Matillion Auth Configuration - */ - connection?: ConfigConnection; - /** - * Tableau API version. If not provided, the version will be used from the tableau server. - * - * Sigma API version. - * - * ThoughtSpot API version to use - * - * OpenMetadata server API version to use. - * - * Airbyte API version. - */ - apiVersion?: string; - /** - * Types of methods used to authenticate to the tableau instance - * - * Choose between different authentication types for Databricks. - * - * Choose Auth Config Type. - * - * Types of methods used to authenticate to the alation instance - * - * Authentication type to connect to Apache Ranger. - */ - authType?: AuthenticationTypeForTableau | NoConfigAuthenticationTypes; - /** - * Pagination limit used while querying the tableau metadata API for getting data sources - * - * Pagination limit used while querying the SAP ERP API for fetching the entities - * - * Pagination limit used for Alation APIs pagination - */ - paginationLimit?: number; - /** - * Proxy URL for the tableau server. If not provided, the hostPort will be used. This is - * used to generate the dashboard & Chart URL. - */ - proxyURL?: string; - /** - * Tableau Site Name. - * - * SharePoint site name - */ - siteName?: string; - /** - * SSL Configuration details. - * - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: SSLConfigObject; - /** - * Boolean marking if we need to verify the SSL certs for Grafana. Default to True. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - * - * Boolean marking if we need to verify the SSL certs for KafkaConnect REST API. True by - * default. - */ - verifySSL?: boolean | VerifySSL; - /** - * Access Token for Mode Dashboard - * - * Access token to connect to DOMO - */ - accessToken?: string; - /** - * Access Token Password for Mode Dashboard - */ - accessTokenPassword?: string; - /** - * Filter query parameter for some of the Mode API calls - */ - filterQueryParam?: string; - /** - * Mode Workspace Name - */ - workspaceName?: string; - connectionOptions?: { [key: string]: string }; - /** - * Source Python Class Name to instantiated by the ingestion workflow - */ - sourcePythonClass?: string; - /** - * API Host to connect to DOMO instance - */ - apiHost?: string; - /** - * URL of your Domo instance, e.g., https://openmetadata.domo.com - */ - instanceDomain?: string; - /** - * Secret Token to connect DOMO - * - * Secret token to connect to DOMO - */ - secretToken?: string; - /** - * AWS Account ID - */ - awsAccountId?: string; - awsConfig?: AWSCredentials; - /** - * The authentication method that the user uses to sign in. - */ - identityType?: IdentityType; - /** - * The Amazon QuickSight namespace that contains the dashboard IDs in this request ( To be - * provided when identityType is `ANONYMOUS` ) - */ - namespace?: string; - certificates?: QlikCertificatesBy; - /** - * Qlik Sense Base URL, used for genrating dashboard & chat url - */ - displayUrl?: string; - /** - * User Directory. - */ - userDirectory?: string; - /** - * User ID. - */ - userId?: string; - /** - * Validate Host Name - */ - validateHostName?: boolean; - /** - * The Project UUID for your Lightdash instance - */ - projectUUID?: string; - /** - * Use if your Lightdash instance is behind a proxy like (Cloud IAP) - */ - proxyAuthentication?: string; - /** - * The Space UUID for your Lightdash instance - */ - spaceUUID?: string; - /** - * Login Mode for Microstrategy's REST API connection. You can authenticate with one of the - * following authentication modes: `Standard (1)`, `Anonymous (8)`. Default will be - * `Standard (1)`. If you're using demo account for Microstrategy, it will be needed to - * authenticate through loginMode `8`. - */ - loginMode?: string; - /** - * MicroStrategy Project Name - * - * Project name to create the refreshToken. Can be anything - */ - projectName?: string; - /** - * Space types of Qlik Cloud to filter the dashboards ingested into the platform. - */ - spaceTypes?: SpaceType[]; - /** - * ThoughtSpot authentication configuration - */ - authentication?: Authenticationation; - /** - * Org ID for multi-tenant ThoughtSpot instances. This is applicable for ThoughtSpot Cloud - * only. - */ - orgId?: string; - /** - * Page size for pagination in API requests. Default is 100. - */ - pageSize?: number; - /** - * Billing Project ID - */ - billingProjectId?: string; - /** - * If using Metastore, Key-Value pairs that will be used to add configs to the SparkSession. - */ - connectionArguments?: { [key: string]: any }; - /** - * Cost per TiB for BigQuery usage - */ - costPerTB?: number; - /** - * GCP Credentials - * - * Azure Credentials - * - * GCP Credentials for Google Drive API - */ - credentials?: GCPCredentials; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ - scheme?: ConfigScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsIncrementalMetadataExtraction?: boolean; - /** - * Supports Lineage Extraction. - */ - supportsLineageExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsSystemProfile?: boolean; - /** - * Supports Usage Extraction. - */ - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource types - */ - tableFilterPattern?: FilterPattern; - /** - * Taxonomy location used to fetch policy tags - */ - taxonomyLocation?: string; - /** - * Project IDs used to fetch policy tags - */ - taxonomyProjectID?: string[]; - /** - * Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can - * pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia - * multi-regions are not yet in GA. - */ - usageLocation?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' - * as the database name. - */ - databaseName?: string; - /** - * S3 Staging Directory. Example: s3://postgres/input/ - */ - s3StagingDir?: string; - /** - * Athena workgroup. - */ - workgroup?: string; - /** - * This parameter determines the mode of authentication for connecting to AzureSQL using - * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If - * 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. This mode is useful for establishing secure and seamless - * connections with AzureSQL. - * - * This parameter determines the mode of authentication for connecting to Azure Synapse - * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. - * If 'Active Directory Integrated' is selected, password is not required as it uses the - * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you - * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing - * secure and seamless connections with Azure Synapse. - */ - authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - * - * Initial Redshift database to connect to. If you want to ingest all databases, set - * ingestAllDatabases to true. - * - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - database?: string; - /** - * SQLAlchemy driver for AzureSQL. - * - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of - * this. - * - * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top - * of this. - */ - ingestAllDatabases?: boolean; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - * - * Optional name to give to the schema in OpenMetadata. If left blank, we will use default - * as the schema name - */ - databaseSchema?: string; - /** - * Clickhouse SQL connection duration. - */ - duration?: number; - /** - * Use HTTPS Protocol for connection with clickhouse - */ - https?: boolean; - /** - * Path to key file for establishing secure connection - */ - keyfile?: string; - /** - * Establish secure connection with clickhouse - */ - secure?: boolean; - /** - * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you - * would like to restrict the metadata reading to a single catalog. When left blank, - * OpenMetadata Ingestion attempts to scan all the catalog. - * - * Presto catalog - * - * Catalog of the data source. - */ - catalog?: IcebergCatalog | string; - /** - * The maximum amount of time (in seconds) to wait for a successful connection to the data - * source. If the connection attempt takes longer than this timeout period, an error will be - * returned. - * - * Connection timeout in seconds. - */ - connectionTimeout?: number | number; - /** - * Databricks compute resources URL. - */ - httpPath?: string; - /** - * Table name to fetch the query history. - */ - queryHistoryTable?: string; - /** - * CLI Driver version to connect to DB2. If not provided, the latest version will be used. - */ - clidriverVersion?: string; - /** - * License to connect to DB2. - */ - license?: string; - /** - * License file name to connect to DB2. - */ - licenseFileName?: string; - supportsViewLineageExtraction?: boolean; - /** - * Available sources to fetch the metadata. - * - * Available sources to fetch files. - * - * Available sources to fetch metadata. - */ - configSource?: DeltaLakeConfigurationSource; - /** - * Authentication mode to connect to hive. - */ - auth?: AuthEnum; - /** - * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. - * - * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. - */ - authOptions?: string; - /** - * If authenticating with Kerberos specify the Kerberos service name - */ - kerberosServiceName?: string; - /** - * Hive Metastore Connection Details - */ - metastoreConnection?: HiveMetastoreConnectionDetails; - /** - * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure - * communication. - * - * Establish secure connection with Impala - */ - useSSL?: boolean; - /** - * Authentication mode to connect to Impala. - */ - authMechanism?: AuthMechanismEnum; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - /** - * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if - * you need to enable thick connection mode. By default, we bring instant client 19 and - * point to /instantclient. - */ - instantClientDirectory?: string; - /** - * Connect with oracle by either passing service name or database schema name. - */ - oracleConnectionType?: OracleConnectionType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - sslMode?: SSLMode; - /** - * Protocol ( Connection Argument ) to connect to Presto. - */ - protocol?: string; - /** - * Verify ( Connection Argument for SSL ) to connect to Presto. - * - * Verify ( Connection Argument for SSL ) to connect to Trino. - */ - verify?: string; - /** - * Salesforce Organization ID is the unique identifier for your Salesforce identity - * - * Snowplow BDP Organization ID - */ - organizationId?: string; - /** - * API version of the Salesforce instance - */ - salesforceApiVersion?: string; - /** - * Domain of Salesforce instance - */ - salesforceDomain?: string; - /** - * Salesforce Security Token. - */ - securityToken?: string; - /** - * Salesforce Object Name. - */ - sobjectName?: string; - /** - * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the - * account is xyz1234.us-east-1.gcp - * - * Specifies an account string to override the default account string defined for the - * database user. Accounts are used by the database for workload management and resource - * usage monitoring. - */ - account?: string; - /** - * Full name of the schema where the account usage data is stored. - */ - accountUsageSchema?: string; - /** - * Optional configuration for ingestion to keep the client session active in case the - * ingestion process runs for longer durations. - */ - clientSessionKeepAlive?: boolean; - /** - * Cost of credit for the Snowflake account. - */ - creditCost?: number; - /** - * Optional configuration for ingestion of streams, By default, it will skip the streams. - */ - includeStreams?: boolean; - /** - * Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the - * TRANSIENT tables. - */ - includeTransientTables?: boolean; - /** - * Connection to Snowflake instance via Private Key - */ - privateKey?: string; - /** - * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user - * should have enough privileges to alter the session. - */ - queryTag?: string; - /** - * Snowflake Role. - */ - role?: string; - /** - * Snowflake Passphrase Key used with Private Key - */ - snowflakePrivatekeyPassphrase?: string; - /** - * Snowflake source host for the Snowflake account. - */ - snowflakeSourceHost?: string; - /** - * Snowflake warehouse. - */ - warehouse?: string; - /** - * Proxies for the connection to Trino data source - */ - proxies?: { [key: string]: string }; - /** - * Pinot Controller Host and Port of the data source. - */ - pinotControllerHost?: string; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - /** - * Couchbase connection Bucket options. - */ - bucket?: string; - /** - * Hostname of the Couchbase service. - */ - hostport?: string; - /** - * Enable dataflow for ingestion - */ - dataflows?: boolean; - /** - * Custom filter for dataflows - */ - dataflowsCustomFilter?: { [key: string]: any } | string; - /** - * Enable datatables for ingestion - */ - datatables?: boolean; - /** - * Custom filter for datatables - */ - dataTablesCustomFilter?: { [key: string]: any } | string; - /** - * Enable report for ingestion - */ - reports?: boolean; - /** - * Custom filter for reports - */ - reportsCustomFilter?: { [key: string]: any } | string; - /** - * Hostname of SAS Viya deployment. - */ - serverHost?: string; - /** - * Table property to look for the Owner. - */ - ownershipProperty?: string; - /** - * Specifies additional data needed by a logon mechanism, such as a secure token, - * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon - * mechanism. - */ - logdata?: string; - /** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ - logmech?: Logmech; - /** - * Specifies the transaction mode for the connection - */ - tmode?: TransactionMode; - /** - * Client SSL/TLS settings. - */ - tls?: SSLTLSSettings; - /** - * HTTP Link for SSAS ACCESS - */ - httpConnection?: string; - /** - * Base URL of the Epic FHIR server - */ - fhirServerUrl?: string; - /** - * FHIR specification version (R4, STU3, DSTU2) - */ - fhirVersion?: FHIRVersion; - /** - * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a - * single default schema will be used. - */ - includeScopes?: boolean; - /** - * If true, both admin and system tables (sys_* tables) will be fetched. If false, only - * admin tables will be fetched. - */ - includeSystemTables?: boolean; - /** - * basic.auth.user.info schema registry config property, Client HTTP credentials in the form - * of username:password. - */ - basicAuthUserInfo?: string; - /** - * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - * - * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 - */ - bootstrapServers?: string; - /** - * Confluent Kafka Consumer Config. From - * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md - * - * Confluent Redpanda Consumer Config - */ - consumerConfig?: { [key: string]: any }; - /** - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - */ - consumerConfigSSL?: ConsumerConfigSSLClass; - /** - * sasl.mechanism Consumer Config property - */ - saslMechanism?: SaslMechanismType; - /** - * sasl.password consumer config property - */ - saslPassword?: string; - /** - * sasl.username consumer config property - */ - saslUsername?: string; - /** - * Confluent Kafka Schema Registry Config. From - * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient - * - * Confluent Redpanda Schema Registry Config. - */ - schemaRegistryConfig?: { [key: string]: any }; - /** - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - */ - schemaRegistrySSL?: ConsumerConfigSSLClass; - /** - * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get - * topic schema from registry. - */ - schemaRegistryTopicSuffixName?: string; - /** - * Confluent Kafka Schema Registry URL. - * - * Confluent Redpanda Schema Registry URL. - */ - schemaRegistryURL?: string; - /** - * security.protocol consumer config property - * - * Kafka security protocol config - */ - securityProtocol?: KafkaSecurityProtocol; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Enable encryption for the Amundsen Neo4j Connection. - */ - encrypted?: boolean; - /** - * Maximum connection lifetime for the Amundsen Neo4j Connection. - */ - maxConnectionLifeTime?: number; - /** - * Enable SSL validation for the Amundsen Neo4j Connection. - */ - validateSSL?: boolean; - /** - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - /** - * List of entities that you need to reindex - */ - entities?: string[]; - recreateIndex?: boolean; - runMode?: RunMode; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: ConfigElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * service type of the data source. - */ - databaseServiceName?: string[]; - /** - * Name of the Entity Type available in Atlas. - */ - entity_type?: string; - /** - * service type of the messaging source - * - * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. - * e.g. local_kafka - */ - messagingServiceName?: string[] | string; - /** - * Custom OpenMetadata Classification name for alation tags. - */ - alationTagClassificationName?: string; - /** - * Specifies if hidden datasources should be included while ingesting. - */ - includeHiddenDatasources?: boolean; - /** - * Specifies if undeployed datasources should be included while ingesting. - */ - includeUndeployedDatasources?: boolean; - /** - * Specifies if Dashboards are to be ingested while running the ingestion job. - */ - ingestDashboards?: boolean; - /** - * Specifies if Datasources are to be ingested while running the ingestion job. - */ - ingestDatasources?: boolean; - /** - * Specifies if Domains are to be ingested while running the ingestion job. - */ - ingestDomains?: boolean; - /** - * Specifies if Knowledge Articles are to be ingested while running the ingestion job. - */ - ingestKnowledgeArticles?: boolean; - /** - * Specifies if Users and Groups are to be ingested while running the ingestion job. - */ - ingestUsersAndGroups?: boolean; - datasourceLinks?: { [key: string]: string }; - /** - * Pipeline Service Number Of Status - */ - numberOfStatus?: number; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Underlying database connection - */ - databaseConnection?: DatabaseConnectionClass; - /** - * Underlying storage connection - */ - packageConnection?: S3Connection | string; - /** - * Fivetran API Secret. - */ - apiSecret?: string; - /** - * Fivetran API Limit For Pagination. - */ - limit?: number; - /** - * URL to the Dagster instance - * - * DBT cloud Access URL. - */ - host?: string; - /** - * Connection Time Limit Between OM and Dagster Graphql API in second - */ - timeout?: number; - /** - * We support username/password or client certificate authentication - */ - nifiConfig?: NifiCredentialsConfiguration; - /** - * Spline UI Host & Port. - */ - uiHostPort?: string; - /** - * service type of the messaging source - */ - brokersUrl?: string; - /** - * consumer group name - */ - consumerGroupName?: string; - /** - * initial Kafka consumer offset - */ - consumerOffsets?: InitialConsumerOffsets; - /** - * max allowed wait time - */ - poolTimeout?: number; - /** - * SASL Configuration details. - */ - saslConfig?: SASLClientConfig; - /** - * max allowed inactivity time - */ - sessionTimeout?: number; - /** - * topic from where Open lineage events will be pulled - */ - topicName?: string; - /** - * We support username/password or No Authentication - */ - KafkaConnectConfig?: UsernamePasswordAuthentication; - /** - * ID of your DBT cloud account - */ - accountId?: string; - /** - * DBT cloud Metadata API URL. - */ - discoveryAPI?: string; - /** - * List of IDs of your DBT cloud jobs seperated by comma `,` - */ - jobIds?: string[]; - /** - * Number of runs to fetch from DBT cloud - */ - numberOfRuns?: number; - /** - * List of IDs of your DBT cloud projects seperated by comma `,` - */ - projectIds?: string[]; - /** - * The name of your azure data factory. - */ - factory_name?: string; - /** - * The name of your resource group the data factory is associated with. - */ - resource_group_name?: string; - /** - * Number of days in the past to filter pipeline runs. - */ - run_filter_days?: number; - /** - * The azure subscription identifier. - */ - subscription_id?: string; - /** - * Cloud provider where Snowplow is deployed - */ - cloudProvider?: CloudProvider; - /** - * Path to pipeline configuration files for Community deployment - */ - configPath?: string; - /** - * Snowplow Console URL for BDP deployment - */ - consoleUrl?: string; - /** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - */ - deployment?: SnowplowDeployment; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Mlflow Model registry backend. E.g., - * mysql+pymysql://mlflow:password@localhost:3307/experiments - */ - registryUri?: string; - /** - * Mlflow Experiment tracking URI. E.g., http://localhost:5000 - */ - trackingUri?: string; - /** - * location/region of google cloud project - */ - location?: string; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Connection Timeout in Seconds - */ - connectionTimeoutSecs?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Email to impersonate using domain-wide delegation - */ - delegatedEmail?: string; - /** - * Specific shared drive ID to connect to - * - * SharePoint drive ID. If not provided, default document library will be used - */ - driveId?: string; - /** - * Extract metadata only for Google Sheets files - */ - includeGoogleSheets?: boolean; - /** - * Include shared/team drives in metadata extraction - */ - includeTeamDrives?: boolean; - /** - * SharePoint site URL - */ - siteUrl?: string; - [property: string]: any; -} - -/** - * We support username/password or No Authentication - * - * username/password auth - */ -export interface UsernamePasswordAuthentication { - /** - * KafkaConnect password to authenticate to the API. - */ - password?: string; - /** - * KafkaConnect user to authenticate to the API. - */ - username?: string; -} - -/** - * Regex to only fetch api collections with names matching the pattern. - * - * Regex to only fetch entities that matches the pattern. - * - * Regex exclude or include charts that matches the pattern. - * - * Regex to exclude or include dashboards that matches the pattern. - * - * Regex exclude or include data models that matches the pattern. - * - * Regex to exclude or include projects that matches the pattern. - * - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only include/exclude tables that matches the pattern. - * - * Regex to include/exclude FHIR resource categories - * - * Regex to include/exclude FHIR resource types - * - * Regex to only fetch topics that matches the pattern. - * - * Regex exclude pipelines. - * - * Regex to only fetch containers that matches the pattern. - * - * Regex to only fetch MlModels with names matching the pattern. - * - * Regex to only fetch search indexes that matches the pattern. - * - * Regex to only fetch databases that matches the pattern. - * - * Regex to only fetch tables or databases that matches the pattern. - * - * Regex to only fetch stored procedures that matches the pattern. - * - * Regex exclude tables or databases that matches the pattern. - * - * Regex to only compute metrics for table that matches the given tag, tiers, gloassary - * pattern. - * - * Regex to only include/exclude directories that matches the pattern. - * - * Regex to only include/exclude files that matches the pattern. - * - * Regex to only include/exclude spreadsheets that matches the pattern. - * - * Regex to only include/exclude worksheets that matches the pattern. - * - * Regex to only fetch tags that matches the pattern. - */ -export interface FilterPattern { - /** - * List of strings/regex patterns to match and exclude only database entities that match. - */ - excludes?: string[]; - /** - * List of strings/regex patterns to match and include only database entities that match. - */ - includes?: string[]; -} - -/** - * Authentication mode to connect to hive. - */ -export enum AuthEnum { - Basic = "BASIC", - Custom = "CUSTOM", - Gssapi = "GSSAPI", - Jwt = "JWT", - Kerberos = "KERBEROS", - LDAP = "LDAP", - None = "NONE", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * Authentication mode to connect to Impala. - */ -export enum AuthMechanismEnum { - Gssapi = "GSSAPI", - Jwt = "JWT", - LDAP = "LDAP", - Nosasl = "NOSASL", - Plain = "PLAIN", -} - -/** - * OpenMetadata Server Authentication Provider. - * - * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as - * the one configured on OpenMetadata server. - */ -export enum AuthProvider { - Auth0 = "auth0", - AwsCognito = "aws-cognito", - Azure = "azure", - Basic = "basic", - CustomOidc = "custom-oidc", - Google = "google", - LDAP = "ldap", - Okta = "okta", - Openmetadata = "openmetadata", - Saml = "saml", -} - -/** - * Types of methods used to authenticate to the tableau instance - * - * Basic Auth Credentials - * - * Access Token Auth Credentials - * - * Choose between different authentication types for Databricks. - * - * Personal Access Token authentication for Databricks. - * - * OAuth2 Machine-to-Machine authentication using Service Principal credentials for - * Databricks. - * - * Azure Active Directory authentication for Azure Databricks workspaces using Service - * Principal. - * - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - * - * Configuration for connecting to DataStax Astra DB in the cloud. - * - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * API Access Token Auth Credentials - * - * Basic Auth Configuration for ElasticSearch - * - * SSL Certificates By Path - * - * AWS credentials configs. - * - * Authentication type to connect to Apache Ranger. - * - * Configuration for connecting to Ranger Basic Auth. - */ -export interface AuthenticationTypeForTableau { - /** - * Password to access the service. - * - * Password to connect to source. - * - * Elastic Search Password for Login - * - * Ranger password to authenticate to the API. - */ - password?: string; - /** - * Username to access the service. - * - * Elastic Search Username for Login - * - * Ranger user to authenticate to the API. - */ - username?: string; - /** - * Personal Access Token Name. - */ - personalAccessTokenName?: string; - /** - * Personal Access Token Secret. - */ - personalAccessTokenSecret?: string; - /** - * Generated Personal Access Token for Databricks workspace authentication. This token is - * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. - */ - token?: string; - /** - * Service Principal Application ID created in your Databricks Account Console for OAuth - * Machine-to-Machine authentication. - */ - clientId?: string; - /** - * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for - * secure OAuth2 authentication. - */ - clientSecret?: string; - /** - * Azure Service Principal Application (client) ID registered in your Azure Active Directory. - */ - azureClientId?: string; - /** - * Azure Service Principal client secret created in Azure AD for authentication. - */ - azureClientSecret?: string; - /** - * Azure Active Directory Tenant ID where your Service Principal is registered. - */ - azureTenantId?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; - /** - * JWT to connect to source. - */ - jwt?: string; - /** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ - cloudConfig?: DataStaxAstraDBConfiguration; - /** - * Access Token for the API - */ - accessToken?: string; - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * AWS credentials configs. - */ -export interface AWSCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface AzureCredentials { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Configuration for connecting to DataStax Astra DB in the cloud. - */ -export interface DataStaxAstraDBConfiguration { - /** - * Timeout in seconds for establishing new connections to Cassandra. - */ - connectTimeout?: number; - /** - * Timeout in seconds for individual Cassandra requests. - */ - requestTimeout?: number; - /** - * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax - * Astra DB. - */ - secureConnectBundle?: string; - /** - * The Astra DB application token used for authentication. - */ - token?: string; - [property: string]: any; -} - -/** - * Database Authentication types not requiring config. - */ -export enum NoConfigAuthenticationTypes { - OAuth2 = "OAuth2", -} - -/** - * ThoughtSpot authentication configuration - * - * Types of methods used to authenticate to the alation instance - * - * Basic Auth Credentials - * - * API Access Token Auth Credentials - */ -export interface Authenticationation { - /** - * Password to access the service. - */ - password?: string; - /** - * Username to access the service. - */ - username?: string; - /** - * Access Token for the API - */ - accessToken?: string; -} - -export interface AuthenticationModeObject { - /** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ - authentication?: Authentication; - /** - * Connection Timeout from Connection String for AzureSQL. - * - * Connection Timeout from Connection String for Azure Synapse. - */ - connectionTimeout?: number; - /** - * Encrypt from Connection String for AzureSQL. - * - * Encrypt from Connection String for Azure Synapse. - */ - encrypt?: boolean; - /** - * Trust Server Certificate from Connection String for AzureSQL. - * - * Trust Server Certificate from Connection String for Azure Synapse. - */ - trustServerCertificate?: boolean; - [property: string]: any; -} - -/** - * Authentication from Connection String for AzureSQL. - * - * Authentication from Connection String for Azure Synapse. - */ -export enum Authentication { - ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", - ActiveDirectoryPassword = "ActiveDirectoryPassword", - ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", -} - -/** - * Iceberg Catalog configuration. - */ -export interface IcebergCatalog { - /** - * Catalog connection configuration, depending on your catalog type. - */ - connection: Connection; - /** - * Custom Database Name for your Iceberg Service. If not set it will be 'default'. - */ - databaseName?: string; - /** - * Catalog Name. - */ - name: string; - /** - * Warehouse Location. Used to specify a custom warehouse location if needed. - */ - warehouseLocation?: string; -} - -/** - * Catalog connection configuration, depending on your catalog type. - * - * Iceberg Hive Catalog configuration. - * - * Iceberg REST Catalog configuration. - * - * Iceberg Glue Catalog configuration. - * - * Iceberg DynamoDB Catalog configuration. - */ -export interface Connection { - fileSystem?: IcebergFileSystem; - /** - * Uri to the Hive Metastore. Example: 'thrift://localhost:9083' - * - * Uri to the REST catalog. Example: 'http://rest-catalog/ws/' - */ - uri?: string; - /** - * OAuth2 credential to use when initializing the catalog. - */ - credential?: OAuth2Credential; - /** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ - sigv4?: Sigv4; - /** - * SSL Configuration details. - */ - ssl?: SSLCertificatesByPath; - /** - * Berarer token to use for the 'Authorization' header. - */ - token?: string; - awsConfig?: AWSCredentials; - /** - * DynamoDB table name. - */ - tableName?: string; -} - -/** - * OAuth2 credential to use when initializing the catalog. - */ -export interface OAuth2Credential { - /** - * OAuth2 Client ID. - */ - clientId?: string; - /** - * OAuth2 Client Secret - */ - clientSecret?: string; -} - -/** - * Iceberg File System configuration, based on where the Iceberg Warehouse is located. - */ -export interface IcebergFileSystem { - type?: Credentials | null; -} - -/** - * AWS credentials configs. - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface Credentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Sign requests to the REST Server using AWS SigV4 protocol. - */ -export interface Sigv4 { - /** - * The service signing name to use when SigV4 signs a request. - */ - signingName?: string; - /** - * AWS Region to use when SigV4 signs a request. - */ - signingRegion?: string; - [property: string]: any; -} - -/** - * SSL Configuration details. - * - * SSL Certificates By Path - */ -export interface SSLCertificatesByPath { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; -} - -/** - * Qlik Authentication Certificate By Values - * - * Qlik Authentication Certificate File Path - */ -export interface QlikCertificatesBy { - sslConfig?: ConsumerConfigSSLClass; - /** - * Client Certificate - */ - clientCertificate?: string; - /** - * Client Key Certificate. - */ - clientKeyCertificate?: string; - /** - * Root Certificate. - */ - rootCertificate?: string; - [property: string]: any; -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - */ -export interface ConsumerConfigSSLClass { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * Cloud provider where Snowplow is deployed - */ -export enum CloudProvider { - Aws = "AWS", - Azure = "Azure", - Gcp = "GCP", -} - -/** - * Available sources to fetch the metadata. - * - * Deltalake Metastore configuration. - * - * DeltaLake Storage Connection Config - * - * Available sources to fetch files. - * - * Local config source where no extra information needs to be sent. - * - * Azure Datalake Storage will ingest files in container - * - * DataLake GCS storage will ingest metadata of files - * - * DataLake S3 bucket will ingest metadata of files in bucket - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface DeltaLakeConfigurationSource { - /** - * pySpark App Name. - */ - appName?: string; - /** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - */ - connection?: ConfigSourceConnection; - /** - * Bucket Name of the data source. - */ - bucketName?: string; - /** - * Prefix of the data source. - */ - prefix?: string; - securityConfig?: SecurityConfigClass; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Metastore connection configuration, depending on your metastore type. - * - * Available sources to fetch files. - * - * DataLake S3 bucket will ingest metadata of files in bucket - */ -export interface ConfigSourceConnection { - /** - * Thrift connection to the metastore service. E.g., localhost:9083 - */ - metastoreHostPort?: string; - /** - * Driver class name for JDBC metastore. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., - * org.mariadb.jdbc.Driver - */ - driverName?: string; - /** - * Class path to JDBC driver required for JDBC connection. The value will be mapped as - * spark.driver.extraClassPath sparks property. - */ - jdbcDriverClassPath?: string; - /** - * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive - */ - metastoreDb?: string; - /** - * Password to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. - */ - password?: string; - /** - * Username to use against metastore database. The value will be mapped as - * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. - */ - username?: string; - /** - * Local path for the local file with metastore data. E.g., /tmp/metastore.db - */ - metastoreFilePath?: string; - securityConfig?: AWSCredentials; -} - -/** - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - * - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * AWS credentials configs. - */ -export interface SecurityConfigClass { - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - * - * Pass the raw credential values provided by GCP - * - * Pass the path of file containing the GCP credentials info - * - * Use the application default credentials - */ -export interface GCPCredentialsConfiguration { - /** - * Google Cloud auth provider certificate. - */ - authProviderX509CertUrl?: string; - /** - * Google Cloud auth uri. - */ - authUri?: string; - /** - * Google Cloud email. - */ - clientEmail?: string; - /** - * Google Cloud Client ID. - */ - clientId?: string; - /** - * Google Cloud client certificate uri. - */ - clientX509CertUrl?: string; - /** - * Google Cloud private key. - */ - privateKey?: string; - /** - * Google Cloud private key id. - */ - privateKeyId?: string; - /** - * Project ID - * - * GCP Project ID to parse metadata from - */ - projectId?: string[] | string; - /** - * Google Cloud token uri. - */ - tokenUri?: string; - /** - * Google Cloud Platform account type. - * - * Google Cloud Platform ADC ( Application Default Credentials ) - */ - type?: string; - /** - * Path of the file containing the GCP credentials info - */ - path?: string; - /** - * Google Security Token Service audience which contains the resource name for the workload - * identity pool and the provider identifier in that pool. - */ - audience?: string; - /** - * This object defines the mechanism used to retrieve the external credential from the local - * environment so that it can be exchanged for a GCP access token via the STS endpoint - */ - credentialSource?: { [key: string]: string }; - /** - * Google Cloud Platform account type. - */ - externalType?: string; - /** - * Google Security Token Service subject token type based on the OAuth 2.0 token exchange - * spec. - */ - subjectTokenType?: string; - /** - * Google Security Token Service token exchange endpoint. - */ - tokenURL?: string; - [property: string]: any; -} - -/** - * we enable the authenticated service account to impersonate another service account - * - * Pass the values to impersonate a service account of Google Cloud - */ -export interface GCPImpersonateServiceAccountValues { - /** - * The impersonated service account email - */ - impersonateServiceAccount?: string; - /** - * Number of seconds the delegated credential should be valid - */ - lifetime?: number; - [property: string]: any; -} - -/** - * Choose between API or database connection fetch metadata from superset. - * - * Superset API Connection Config - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - * - * Choose between Database connection or HDB User Store connection. - * - * Sap Hana Database SQL Connection Config - * - * Sap Hana Database HDB User Store Connection Config - * - * Choose between mysql and postgres connection for alation database - * - * Underlying database connection. See - * https://airflow.apache.org/docs/apache-airflow/stable/howto/set-up-database.html for - * supported backends. - * - * Lineage Backend Connection Config - * - * SQLite Database Connection Config - * - * Matillion Auth Configuration - * - * Matillion ETL Auth Config. - */ -export interface ConfigConnection { - /** - * Password for Superset. - * - * Password to connect to Hana. - * - * Password to connect to SQLite. Blank for in-memory database. - * - * Password to connect to the Matillion. - */ - password?: string; - /** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ - provider?: Provider; - /** - * SSL Configuration details. - */ - sslConfig?: ConnectionSSLConfig; - /** - * Username for Superset. - * - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - * - * Username to connect to Hana. This user should have privileges to read all the metadata. - * - * Username to connect to SQLite. Blank for in-memory database. - * - * Username to connect to the Matillion. This user should have privileges to read all the - * metadata in Matillion. - */ - username?: string; - verifySSL?: VerifySSL; - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - * - * Database of the data source. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - * - * Host and port of the Hana service. - * - * Host and port of the SQLite service. Blank for in-memory database. - * - * Matillion Host - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: ConnectionScheme; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: ConnectionType; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - * - * Database Schema of the data source. This is an optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; - /** - * HDB Store User Key generated from the command `hdbuserstore SET - * ` - */ - userKey?: string; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * How to run the SQLite database. :memory: by default. - */ - databaseMode?: string; - supportsViewLineageExtraction?: boolean; -} - -/** - * Choose Auth Config Type. - * - * Common Database Connection Config - * - * IAM Auth Database Connection Config - * - * Azure Database Connection Config - */ -export interface AuthConfigurationType { - /** - * Password to connect to source. - */ - password?: string; - awsConfig?: AWSCredentials; - azureConfig?: AzureCredentials; -} - -/** - * Authentication provider for the Superset service. For basic user/password authentication, - * the default value `db` can be used. This parameter is used internally to connect to - * Superset's REST API. - */ -export enum Provider { - DB = "db", - LDAP = "ldap", -} - -/** - * Storage config to store sample data - */ -export interface SampleDataStorageConfig { - config?: DataStorageConfig; -} - -/** - * Storage config to store sample data - */ -export interface DataStorageConfig { - /** - * Bucket Name - */ - bucketName?: string; - /** - * Provide the pattern of the path where the generated sample data file needs to be stored. - */ - filePathPattern?: string; - /** - * When this field enabled a single parquet file will be created to store sample data, - * otherwise we will create a new file per day - */ - overwriteData?: boolean; - /** - * Prefix of the data source. - */ - prefix?: string; - storageConfig?: AwsCredentials; - [property: string]: any; -} - -/** - * AWS credentials configs. - */ -export interface AwsCredentials { - /** - * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume - * Role - */ - assumeRoleArn?: string; - /** - * An identifier for the assumed role session. Use the role session name to uniquely - * identify a session when the same role is assumed by different principals or for different - * reasons. Required Field in case of Assume Role - */ - assumeRoleSessionName?: string; - /** - * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume - * Role - */ - assumeRoleSourceIdentity?: string; - /** - * AWS Access key ID. - */ - awsAccessKeyId?: string; - /** - * AWS Region - */ - awsRegion?: string; - /** - * AWS Secret Access Key. - */ - awsSecretAccessKey?: string; - /** - * AWS Session Token. - */ - awsSessionToken?: string; - /** - * EndPoint URL for the AWS - */ - endPointURL?: string; - /** - * The name of a profile to use with the boto session. - */ - profileName?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum ConnectionScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", - SqlitePysqlite = "sqlite+pysqlite", -} - -/** - * Client SSL configuration - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - */ -export interface ConnectionSSLConfig { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; -} - -/** - * SSL Mode to connect to database. - */ -export enum SSLMode { - Allow = "allow", - Disable = "disable", - Prefer = "prefer", - Require = "require", - VerifyCA = "verify-ca", - VerifyFull = "verify-full", -} - -/** - * Service Type - * - * Service type. - */ -export enum ConnectionType { - Backend = "Backend", - MatillionETL = "MatillionETL", - Mysql = "Mysql", - Postgres = "Postgres", - SQLite = "SQLite", -} - -/** - * Client SSL verification. Make sure to configure the SSLConfig if enabled. - * - * Flag to verify SSL Certificate for OpenMetadata Server. - */ -export enum VerifySSL { - Ignore = "ignore", - NoSSL = "no-ssl", - Validate = "validate", -} - -/** - * initial Kafka consumer offset - */ -export enum InitialConsumerOffsets { - Earliest = "earliest", - Latest = "latest", -} - -/** - * GCP credentials configs. - * - * GCP Credentials - * - * GCP Credentials for Google Drive API - * - * Azure Cloud Credentials - * - * Available sources to fetch metadata. - * - * Azure Credentials - */ -export interface GCPCredentials { - /** - * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP - * Credentials Path - */ - gcpConfig?: GCPCredentialsConfiguration; - /** - * we enable the authenticated service account to impersonate another service account - */ - gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; - /** - * Account Name of your storage account - */ - accountName?: string; - /** - * Your Service Principal App ID (Client ID) - */ - clientId?: string; - /** - * Your Service Principal Password (Client Secret) - */ - clientSecret?: string; - /** - * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default - */ - scopes?: string; - /** - * Tenant ID of your Azure Subscription - */ - tenantId?: string; - /** - * Key Vault Name - */ - vaultName?: string; -} - -/** - * Underlying database connection - * - * Mssql Database Connection Config - */ -export interface DatabaseConnectionClass { - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * ODBC driver version in case of pyodbc connection. - */ - driver?: string; - /** - * Host and port of the MSSQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. - */ - ingestAllDatabases?: boolean; - /** - * Password to connect to MSSQL. - */ - password?: string; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: MssqlScheme; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: MssqlType; - /** - * Username to connect to MSSQL. This user should have privileges to read all the metadata - * in MsSQL. - */ - username?: string; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum MssqlScheme { - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", -} - -/** - * Service Type - * - * Service type. - */ -export enum MssqlType { - Mssql = "Mssql", -} - -/** - * Snowplow deployment type (BDP for managed or Community for self-hosted) - * - * Snowplow deployment type - */ -export enum SnowplowDeployment { - Bdp = "BDP", - Community = "Community", -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface ConfigElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * FHIR specification version (R4, STU3, DSTU2) - */ -export enum FHIRVersion { - Dstu2 = "DSTU2", - R4 = "R4", - Stu3 = "STU3", -} - -/** - * Credentials to extract the .lkml files from a repository. This is required to get all the - * lineage and definitions. - * - * Do not set any credentials. Note that credentials are required to extract .lkml views and - * their lineage. - * - * Credentials for a GitHub repository - * - * Credentials for a BitBucket repository - * - * Credentials for a Gitlab repository - */ -export interface GitHubCredentials { - repositoryName?: string; - repositoryOwner?: string; - token?: string; - /** - * Credentials Type - */ - type?: GitHubCredentialsType; - /** - * Main production branch of the repository. E.g., `main` - */ - branch?: string; -} - -/** - * Credentials Type - * - * GitHub Credentials type - * - * BitBucket Credentials type - * - * Gitlab Credentials type - */ -export enum GitHubCredentialsType { - BitBucket = "BitBucket", - GitHub = "GitHub", - Gitlab = "Gitlab", -} - -/** - * The authentication method that the user uses to sign in. - */ -export enum IdentityType { - Anonymous = "ANONYMOUS", - Iam = "IAM", - Quicksight = "QUICKSIGHT", -} - -/** - * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, - * LDAP, KRB5 for Kerberos, or TDNEGO - */ -export enum Logmech { - Custom = "CUSTOM", - Jwt = "JWT", - Krb5 = "KRB5", - LDAP = "LDAP", - Td2 = "TD2", - Tdnego = "TDNEGO", -} - -/** - * Hive Metastore Connection Details - * - * Postgres Database Connection Config - * - * Mysql Database Connection Config - */ -export interface HiveMetastoreConnectionDetails { - /** - * Choose Auth Config Type. - */ - authType?: AuthConfigurationType; - /** - * Custom OpenMetadata Classification name for Postgres policy tags. - */ - classificationName?: string; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Database of the data source. This is optional parameter, if you would like to restrict - * the metadata reading to a single database. When left blank, OpenMetadata Ingestion - * attempts to scan all the databases. - */ - database?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Host and port of the source service. - * - * Host and port of the MySQL service. - */ - hostPort?: string; - /** - * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of - * this. - */ - ingestAllDatabases?: boolean; - sampleDataStorageConfig?: SampleDataStorageConfig; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * SQLAlchemy driver scheme options. - */ - scheme?: HiveMetastoreConnectionDetailsScheme; - /** - * SSL Configuration details. - */ - sslConfig?: ConsumerConfigSSLClass; - sslMode?: SSLMode; - supportsDatabase?: boolean; - supportsDataDiff?: boolean; - supportsDBTExtraction?: boolean; - supportsLineageExtraction?: boolean; - supportsMetadataExtraction?: boolean; - supportsProfiler?: boolean; - supportsQueryComment?: boolean; - supportsUsageExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: HiveMetastoreConnectionDetailsType; - /** - * Username to connect to Postgres. This user should have privileges to read all the - * metadata in Postgres. - * - * Username to connect to MySQL. This user should have privileges to read all the metadata - * in Mysql. - */ - username?: string; - /** - * Optional name to give to the database in OpenMetadata. If left blank, we will use default - * as the database name. - */ - databaseName?: string; - /** - * Database Schema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion - * attempts to scan all the schemas. - */ - databaseSchema?: string; - /** - * Use slow logs to extract lineage. - */ - useSlowLogs?: boolean; -} - -/** - * SQLAlchemy driver scheme options. - */ -export enum HiveMetastoreConnectionDetailsScheme { - MysqlPymysql = "mysql+pymysql", - PgspiderPsycopg2 = "pgspider+psycopg2", - PostgresqlPsycopg2 = "postgresql+psycopg2", -} - -/** - * Service Type - * - * Service type. - */ -export enum HiveMetastoreConnectionDetailsType { - Mysql = "Mysql", - Postgres = "Postgres", -} - -/** - * We support username/password or client certificate authentication - * - * Configuration for connecting to Nifi Basic Auth. - * - * Configuration for connecting to Nifi Client Certificate Auth. - */ -export interface NifiCredentialsConfiguration { - /** - * Nifi password to authenticate to the API. - */ - password?: string; - /** - * Nifi user to authenticate to the API. - */ - username?: string; - /** - * Boolean marking if we need to verify the SSL certs for Nifi. False by default. - */ - verifySSL?: boolean; - /** - * Path to the root CA certificate - */ - certificateAuthorityPath?: string; - /** - * Path to the client certificate - */ - clientCertificatePath?: string; - /** - * Path to the client key - */ - clientkeyPath?: string; -} - -/** - * Connect with oracle by either passing service name or database schema name. - */ -export interface OracleConnectionType { - /** - * databaseSchema of the data source. This is optional parameter, if you would like to - * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata - * Ingestion attempts to scan all the databaseSchema. - */ - databaseSchema?: string; - /** - * The Oracle Service name is the TNS alias that you give when you remotely connect to your - * database. - */ - oracleServiceName?: string; - /** - * Pass the full constructed TNS string, e.g., - * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). - */ - oracleTNSConnection?: string; - [property: string]: any; -} - -/** - * S3 Connection. - */ -export interface S3Connection { - awsConfig: AWSCredentials; - /** - * Bucket Names of the data source. - */ - bucketNames?: string[]; - connectionArguments?: { [key: string]: any }; - connectionOptions?: { [key: string]: string }; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: S3Type; -} - -/** - * Service Type - * - * S3 service type - */ -export enum S3Type { - S3 = "S3", -} - -/** - * Source to get the .pbit files to extract lineage information - * - * Local config source where no extra information needs to be sent. - * - * Azure storage config for pbit files - * - * GCS storage config for pbit files - * - * S3 storage config for pbit files - */ -export interface PowerBIPbitFilesSource { - /** - * Directory path for the pbit files - */ - path?: string; - /** - * pbit File Configuration type - */ - pbitFileConfigType?: PbitFileConfigType; - /** - * Path of the folder where the .pbit files will be unzipped and datamodel schema will be - * extracted - */ - pbitFilesExtractDir?: string; - prefixConfig?: BucketDetails; - securityConfig?: SecurityConfigClass; -} - -/** - * pbit File Configuration type - */ -export enum PbitFileConfigType { - Azure = "azure", - Gcs = "gcs", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the .pbit files are stored - */ -export interface BucketDetails { - /** - * Name of the bucket where the .pbit files are stored - */ - bucketName?: string; - /** - * Path of the folder where the .pbit files are stored - */ - objectPrefix?: string; -} - -/** - * This schema publisher run modes. - */ -export enum RunMode { - Batch = "batch", - Stream = "stream", -} - -/** - * SASL Configuration details. - * - * SASL client configuration. - */ -export interface SASLClientConfig { - /** - * SASL security mechanism - */ - saslMechanism?: SaslMechanismType; - /** - * The SASL authentication password. - */ - saslPassword?: string; - /** - * The SASL authentication username. - */ - saslUsername?: string; -} - -/** - * sasl.mechanism Consumer Config property - * - * SASL Mechanism consumer config property - * - * SASL security mechanism - */ -export enum SaslMechanismType { - Gssapi = "GSSAPI", - Oauthbearer = "OAUTHBEARER", - Plain = "PLAIN", - ScramSHA256 = "SCRAM-SHA-256", - ScramSHA512 = "SCRAM-SHA-512", -} - -/** - * SQLAlchemy driver scheme options. - * - * Mongo connection scheme options. - * - * Couchbase driver scheme options. - */ -export enum ConfigScheme { - AwsathenaREST = "awsathena+rest", - Bigquery = "bigquery", - ClickhouseHTTP = "clickhouse+http", - ClickhouseNative = "clickhouse+native", - CockroachdbPsycopg2 = "cockroachdb+psycopg2", - Couchbase = "couchbase", - DatabricksConnector = "databricks+connector", - Db2IBMDB = "db2+ibm_db", - Doris = "doris", - Druid = "druid", - ExaWebsocket = "exa+websocket", - Hana = "hana", - Hive = "hive", - HiveHTTP = "hive+http", - HiveHTTPS = "hive+https", - Ibmi = "ibmi", - Impala = "impala", - Impala4 = "impala4", - Mongodb = "mongodb", - MongodbSrv = "mongodb+srv", - MssqlPymssql = "mssql+pymssql", - MssqlPyodbc = "mssql+pyodbc", - MssqlPytds = "mssql+pytds", - MysqlPymysql = "mysql+pymysql", - OracleCxOracle = "oracle+cx_oracle", - PgspiderPsycopg2 = "pgspider+psycopg2", - Pinot = "pinot", - PinotHTTP = "pinot+http", - PinotHTTPS = "pinot+https", - PostgresqlPsycopg2 = "postgresql+psycopg2", - Presto = "presto", - RedshiftPsycopg2 = "redshift+psycopg2", - Snowflake = "snowflake", - SqlitePysqlite = "sqlite+pysqlite", - Teradatasql = "teradatasql", - Trino = "trino", - VerticaVerticaPython = "vertica+vertica_python", -} - -/** - * Recreate Indexes with updated Language - * - * This schema defines the language options available for search index mappings. - */ -export enum SearchIndexMappingLanguage { - En = "EN", - Jp = "JP", - Ru = "RU", - Zh = "ZH", -} - -/** - * Secrets Manager Loader for the Pipeline Service Client. - * - * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager - * Credentials should be loaded from the environment. - */ -export enum SecretsManagerClientLoader { - Airflow = "airflow", - Env = "env", - Noop = "noop", -} - -/** - * Secrets Manager Provider for OpenMetadata Server. - * - * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager - * providers as the ones configured on the OpenMetadata server. - */ -export enum SecretsManagerProvider { - Aws = "aws", - AwsSsm = "aws-ssm", - AzureKv = "azure-kv", - DB = "db", - Gcp = "gcp", - InMemory = "in-memory", - Kubernetes = "kubernetes", - ManagedAws = "managed-aws", - ManagedAwsSsm = "managed-aws-ssm", - ManagedAzureKv = "managed-azure-kv", -} - -/** - * OpenMetadata Client security configuration. - * - * openMetadataJWTClientConfig security configs. - */ -export interface OpenMetadataJWTClientConfig { - /** - * OpenMetadata generated JWT token. - */ - jwtToken: string; -} - -/** - * security.protocol consumer config property - * - * Kafka security protocol config - */ -export enum KafkaSecurityProtocol { - Plaintext = "PLAINTEXT", - SSL = "SSL", - SaslPlaintext = "SASL_PLAINTEXT", - SaslSSL = "SASL_SSL", -} - -export enum SpaceType { - Data = "Data", - Managed = "Managed", - Personal = "Personal", - Shared = "Shared", -} - -/** - * Client SSL configuration - * - * SSL Configuration details. - * - * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config - * connection. - * - * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry - * connection. - * - * SSL Configuration for OpenMetadata Server - * - * OpenMetadata Client configured to validate SSL certificates. - * - * SSL Config - */ -export interface SSLConfigObject { - /** - * The CA certificate used for SSL validation. - */ - caCertificate?: string; - /** - * The SSL certificate used for client authentication. - */ - sslCertificate?: string; - /** - * The private key associated with the SSL certificate. - */ - sslKey?: string; - /** - * SSL Certificates - */ - certificates?: SSLCertificates; - [property: string]: any; -} - -/** - * SSL Certificates - * - * SSL Configuration details. - * - * SSL Certificates By Path - * - * SSL Certificates By Values - */ -export interface SSLCertificates { - /** - * CA Certificate Path - */ - caCertPath?: string; - /** - * Client Certificate Path - */ - clientCertPath?: string; - /** - * Private Key Path - */ - privateKeyPath?: string; - /** - * CA Certificate Value - */ - caCertValue?: string; - /** - * Client Certificate Value - */ - clientCertValue?: string; - /** - * Private Key Value - */ - privateKeyValue?: string; - /** - * Staging Directory Path - */ - stagingDir?: string; -} - -/** - * Client SSL/TLS settings. - */ -export enum SSLTLSSettings { - DisableTLS = "disable-tls", - IgnoreCertificate = "ignore-certificate", - ValidateCertificate = "validate-certificate", -} - -/** - * Specifies the transaction mode for the connection - */ -export enum TransactionMode { - ANSI = "ANSI", - Default = "DEFAULT", - Tera = "TERA", -} - -/** - * REST API Type - * - * REST API type - * - * Service Type - * - * Looker service type - * - * Metabase service type - * - * PowerBI service type - * - * PowerBIReportServer service type - * - * Redash service type - * - * Superset service type - * - * Tableau service type - * - * Mode service type - * - * Custom dashboard service type - * - * service type - * - * QuickSight service type - * - * Qlik sense service type - * - * Lightdash service type - * - * MicroStrategy service type - * - * Qlik Cloud service type - * - * Sigma service type - * - * ThoughtSpot service type - * - * Grafana service type - * - * Service type. - * - * Custom database service type - * - * Kafka service type - * - * Redpanda service type - * - * Custom messaging service type - * - * Amundsen service type - * - * Metadata to Elastic Search type - * - * OpenMetadata service type - * - * Custom pipeline service type - * - * Custom Ml model service type - * - * S3 service type - * - * ADLS service type - * - * Gcs service type - * - * Custom storage service type - * - * ElasticSearch Type - * - * ElasticSearch service type - * - * OpenSearch Type - * - * OpenSearch service type - * - * Custom search service type - * - * Apache Ranger service type - * - * Google Drive service type - * - * SharePoint service type - * - * Custom Drive service type - */ -export enum PurpleType { - Adls = "ADLS", - Airbyte = "Airbyte", - Airflow = "Airflow", - Alation = "Alation", - AlationSink = "AlationSink", - Amundsen = "Amundsen", - Athena = "Athena", - Atlas = "Atlas", - AzureSQL = "AzureSQL", - BigQuery = "BigQuery", - BigTable = "BigTable", - Cassandra = "Cassandra", - Clickhouse = "Clickhouse", - Cockroach = "Cockroach", - Couchbase = "Couchbase", - CustomDashboard = "CustomDashboard", - CustomDatabase = "CustomDatabase", - CustomDrive = "CustomDrive", - CustomMessaging = "CustomMessaging", - CustomMlModel = "CustomMlModel", - CustomPipeline = "CustomPipeline", - CustomSearch = "CustomSearch", - CustomStorage = "CustomStorage", - DBTCloud = "DBTCloud", - Dagster = "Dagster", - DataFactory = "DataFactory", - Databricks = "Databricks", - DatabricksPipeline = "DatabricksPipeline", - Datalake = "Datalake", - Db2 = "Db2", - DeltaLake = "DeltaLake", - DomoDashboard = "DomoDashboard", - DomoDatabase = "DomoDatabase", - DomoPipeline = "DomoPipeline", - Doris = "Doris", - Druid = "Druid", - DynamoDB = "DynamoDB", - ElasticSearch = "ElasticSearch", - Epic = "Epic", - Exasol = "Exasol", - Fivetran = "Fivetran", - Flink = "Flink", - Gcs = "GCS", - Glue = "Glue", - GluePipeline = "GluePipeline", - GoogleDrive = "GoogleDrive", - Grafana = "Grafana", - Greenplum = "Greenplum", - Hive = "Hive", - Iceberg = "Iceberg", - Impala = "Impala", - Kafka = "Kafka", - KafkaConnect = "KafkaConnect", - Kinesis = "Kinesis", - Lightdash = "Lightdash", - Looker = "Looker", - MariaDB = "MariaDB", - Matillion = "Matillion", - Metabase = "Metabase", - MetadataES = "MetadataES", - MicroStrategy = "MicroStrategy", - Mlflow = "Mlflow", - Mode = "Mode", - MongoDB = "MongoDB", - Mssql = "Mssql", - Mysql = "Mysql", - Nifi = "Nifi", - OpenLineage = "OpenLineage", - OpenMetadata = "OpenMetadata", - OpenSearch = "OpenSearch", - Oracle = "Oracle", - PinotDB = "PinotDB", - Postgres = "Postgres", - PowerBI = "PowerBI", - PowerBIReportServer = "PowerBIReportServer", - Presto = "Presto", - QlikCloud = "QlikCloud", - QlikSense = "QlikSense", - QuickSight = "QuickSight", - REST = "Rest", - Ranger = "Ranger", - Redash = "Redash", - Redpanda = "Redpanda", - Redshift = "Redshift", - S3 = "S3", - SAS = "SAS", - SQLite = "SQLite", - SageMaker = "SageMaker", - Salesforce = "Salesforce", - SapERP = "SapErp", - SapHana = "SapHana", - ServiceNow = "ServiceNow", - SharePoint = "SharePoint", - Sigma = "Sigma", - SingleStore = "SingleStore", - Sklearn = "Sklearn", - Snowflake = "Snowflake", - Snowplow = "Snowplow", - Spark = "Spark", - Spline = "Spline", - Ssas = "SSAS", - Ssis = "SSIS", - Stitch = "Stitch", - Superset = "Superset", - Synapse = "Synapse", - Tableau = "Tableau", - Teradata = "Teradata", - ThoughtSpot = "ThoughtSpot", - Trino = "Trino", - UnityCatalog = "UnityCatalog", - VertexAI = "VertexAI", - Vertica = "Vertica", - Wherescape = "Wherescape", -} - -/** - * Additional connection configuration. - */ -export interface SourceConfig { - config?: Pipeline; -} - -/** - * DatabaseService Metadata Pipeline Configuration. - * - * DatabaseService Query Usage Pipeline Configuration. - * - * DatabaseService Query Lineage Pipeline Configuration. - * - * DashboardService Metadata Pipeline Configuration. - * - * MessagingService Metadata Pipeline Configuration. - * - * DatabaseService Profiler Pipeline Configuration. - * - * DatabaseService AutoClassification & Auto Classification Pipeline Configuration. - * - * PipelineService Metadata Pipeline Configuration. - * - * MlModelService Metadata Pipeline Configuration. - * - * StorageService Metadata Pipeline Configuration. - * - * DriveService Metadata Pipeline Configuration. - * - * SearchService Metadata Pipeline Configuration. - * - * TestSuite Pipeline Configuration. - * - * Data Insight Pipeline Configuration. - * - * DBT Pipeline Configuration. - * - * Application Pipeline Configuration. - * - * ApiService Metadata Pipeline Configuration. - * - * Apply a set of operations on a service - */ -export interface Pipeline { - /** - * Regex to only include/exclude databases that matches the pattern. - * - * Regex to only fetch databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Optional configuration to toggle the DDL Statements ingestion. - */ - includeDDL?: boolean; - /** - * Set the 'Include Owners' toggle to control whether to include owners to the ingested - * entity if the owner email matches with a user stored in the OM server as part of metadata - * ingestion. If the ingested entity already exists and has an owner, the owner will not be - * overwritten. - * - * Enabling a flag will replace the current owner with a new owner from the source during - * metadata ingestion, if the current owner is null. It is recommended to keep the flag - * enabled to obtain the owner information during the first metadata ingestion. - */ - includeOwners?: boolean; - /** - * Optional configuration to toggle the Stored Procedures ingestion. - */ - includeStoredProcedures?: boolean; - /** - * Optional configuration to turn off fetching metadata for tables. - */ - includeTables?: boolean; - /** - * Optional configuration to toggle the tags ingestion. - */ - includeTags?: boolean; - /** - * Optional configuration to turn off fetching metadata for views. - */ - includeViews?: boolean; - /** - * Use incremental Metadata extraction after the first execution. This is commonly done by - * getting the changes from Audit tables on the supporting databases. - */ - incremental?: IncrementalMetadataExtractionConfiguration; - /** - * Optional configuration to soft delete databases in OpenMetadata if the source databases - * are deleted. Also, if the database is deleted, all the associated entities like schemas, - * tables, views, stored procedures, lineage, etc., with that database will be deleted - */ - markDeletedDatabases?: boolean; - /** - * Optional configuration to soft delete schemas in OpenMetadata if the source schemas are - * deleted. Also, if the schema is deleted, all the associated entities like tables, views, - * stored procedures, lineage, etc., with that schema will be deleted - */ - markDeletedSchemas?: boolean; - /** - * Optional configuration to soft delete stored procedures in OpenMetadata if the source - * stored procedures are deleted. Also, if the stored procedures is deleted, all the - * associated entities like lineage, etc., with that stored procedures will be deleted - */ - markDeletedStoredProcedures?: boolean; - /** - * This is an optional configuration for enabling soft deletion of tables. When this option - * is enabled, only tables that have been deleted from the source will be soft deleted, and - * this will apply solely to the schema that is currently being ingested via the pipeline. - * Any related entities such as test suites or lineage information that were associated with - * those tables will also be deleted. - */ - markDeletedTables?: boolean; - /** - * Set the 'Override Metadata' toggle to control whether to override the existing metadata - * in the OpenMetadata server with the metadata fetched from the source. If the toggle is - * set to true, the metadata fetched from the source will override the existing metadata in - * the OpenMetadata server. If the toggle is set to false, the metadata fetched from the - * source will not override the existing metadata in the OpenMetadata server. This is - * applicable for fields like description, tags, owner and displayName - */ - overrideMetadata?: boolean; - /** - * Configuration to tune how far we want to look back in query logs to process Stored - * Procedures results. - * - * Configuration to tune how far we want to look back in query logs to process usage data. - * - * Configuration to tune how far we want to look back in query logs to process lineage data. - */ - queryLogDuration?: number; - /** - * Configuration to set the timeout for parsing the query in seconds. - */ - queryParsingTimeoutLimit?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - * - * Regex to only fetch tables or databases that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude tables that matches the pattern. - * - * Regex exclude tables or databases that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Number of Threads to use in order to parallelize Table ingestion. - * - * Number of Threads to use in order to parallelize lineage ingestion. - * - * Number of Threads to use in order to parallelize Drive ingestion. - */ - threads?: number; - /** - * Pipeline type - */ - type?: FluffyType; - /** - * Regex will be applied on fully qualified name (e.g - * service_name.db_name.schema_name.table_name) instead of raw name (e.g. table_name) - * - * Regex will be applied on fully qualified name (e.g service_name.directory_name.file_name) - * instead of raw name (e.g. file_name) - */ - useFqnForFiltering?: boolean; - /** - * Configuration the condition to filter the query history. - */ - filterCondition?: string; - /** - * Configuration to process query cost - */ - processQueryCostAnalysis?: boolean; - /** - * Configuration to set the file path for query logs - */ - queryLogFilePath?: string; - /** - * Configuration to set the limit for query logs - */ - resultLimit?: number; - /** - * Temporary file name to store the query logs before processing. Absolute file path - * required. - */ - stageFileLocation?: string; - /** - * Set 'Cross Database Service Names' to process lineage with the database. - */ - crossDatabaseServiceNames?: string[]; - /** - * Handle Lineage for Snowflake Temporary and Transient Tables. - */ - enableTempTableLineage?: boolean; - /** - * Set the 'Incremental Lineage Processing' toggle to control whether to process lineage - * incrementally. - */ - incrementalLineageProcessing?: boolean; - /** - * Set the 'Override View Lineage' toggle to control whether to override the existing view - * lineage. - */ - overrideViewLineage?: boolean; - /** - * Configuration to set the timeout for parsing the query in seconds. - */ - parsingTimeoutLimit?: number; - /** - * Set the 'Process Cross Database Lineage' toggle to control whether to process table - * lineage across different databases. - */ - processCrossDatabaseLineage?: boolean; - /** - * Set the 'Process Query Lineage' toggle to control whether to process query lineage. - */ - processQueryLineage?: boolean; - /** - * Set the 'Process Stored ProcedureLog Lineage' toggle to control whether to process stored - * procedure lineage. - */ - processStoredProcedureLineage?: boolean; - /** - * Set the 'Process View Lineage' toggle to control whether to process view lineage. - */ - processViewLineage?: boolean; - /** - * Regex to only fetch stored procedures that matches the pattern. - */ - storedProcedureFilterPattern?: FilterPattern; - /** - * Regex exclude or include charts that matches the pattern. - */ - chartFilterPattern?: FilterPattern; - /** - * Regex to exclude or include dashboards that matches the pattern. - */ - dashboardFilterPattern?: FilterPattern; - /** - * Regex exclude or include data models that matches the pattern. - */ - dataModelFilterPattern?: FilterPattern; - /** - * Optional configuration to toggle the ingestion of data models. - */ - includeDataModels?: boolean; - /** - * Optional Configuration to include/exclude draft dashboards. By default it will include - * draft dashboards - */ - includeDraftDashboard?: boolean; - /** - * Details required to generate Lineage - */ - lineageInformation?: LineageInformation; - /** - * Optional configuration to soft delete dashboards in OpenMetadata if the source dashboards - * are deleted. Also, if the dashboard is deleted, all the associated entities like lineage, - * etc., with that dashboard will be deleted - */ - markDeletedDashboards?: boolean; - /** - * Optional configuration to soft delete data models in OpenMetadata if the source data - * models are deleted. Also, if the data models is deleted, all the associated entities like - * lineage, etc., with that data models will be deleted - */ - markDeletedDataModels?: boolean; - /** - * Set the 'Override Lineage' toggle to control whether to override the existing lineage. - */ - overrideLineage?: boolean; - /** - * Regex to exclude or include projects that matches the pattern. - */ - projectFilterPattern?: FilterPattern; - /** - * Option to turn on/off generating sample data during metadata extraction. - */ - generateSampleData?: boolean; - /** - * Optional configuration to soft delete topics in OpenMetadata if the source topics are - * deleted. Also, if the topic is deleted, all the associated entities like sample data, - * lineage, etc., with that topic will be deleted - */ - markDeletedTopics?: boolean; - /** - * Regex to only fetch topics that matches the pattern. - */ - topicFilterPattern?: FilterPattern; - /** - * Regex to only compute metrics for table that matches the given tag, tiers, gloassary - * pattern. - */ - classificationFilterPattern?: FilterPattern; - /** - * Option to turn on/off column metric computation. If enabled, profiler will compute column - * level metrics. - */ - computeColumnMetrics?: boolean; - /** - * Option to turn on/off computing profiler metrics. - */ - computeMetrics?: boolean; - /** - * Option to turn on/off table metric computation. If enabled, profiler will compute table - * level metrics. - */ - computeTableMetrics?: boolean; - processingEngine?: ProcessingEngine; - /** - * Percentage of data or no. of rows used to compute the profiler metrics and run data - * quality tests - * - * Percentage of data or no. of rows we want to execute the profiler and tests on - */ - profileSample?: number; - profileSampleType?: ProfileSampleType; - /** - * Whether to randomize the sample data or not. - */ - randomizedSample?: boolean; - samplingMethodType?: SamplingMethodType; - /** - * Number of threads to use during metric computations - */ - threadCount?: number; - /** - * Profiler Timeout in Seconds - */ - timeoutSeconds?: number; - /** - * Use system tables to extract metrics. Metrics that cannot be gathered from system tables - * will use the default methods. Using system tables can be faster but requires gathering - * statistics before running (for example using the ANALYZE procedure). More information can - * be found in the documentation: https://docs.openmetadata.org/latest/profler - */ - useStatistics?: boolean; - /** - * Set the Confidence value for which you want the column to be tagged as PII. Confidence - * value ranges from 0 to 100. A higher number will yield less false positives but more - * false negatives. A lower number will yield more false positives but less false negatives. - */ - confidence?: number; - /** - * Optional configuration to automatically tag columns that might contain sensitive - * information - */ - enableAutoClassification?: boolean; - /** - * Number of sample rows to ingest when 'Generate Sample Data' is enabled - */ - sampleDataCount?: number; - /** - * Option to turn on/off storing sample data. If enabled, we will ingest sample data for - * each table. - */ - storeSampleData?: boolean; - /** - * Optional configuration to turn off fetching lineage from pipelines. - */ - includeLineage?: boolean; - /** - * Optional configuration to toggle whether the un-deployed pipelines should be ingested or - * not. If set to false, only deployed pipelines will be ingested. - */ - includeUnDeployedPipelines?: boolean; - /** - * Optional configuration to soft delete Pipelines in OpenMetadata if the source Pipelines - * are deleted. Also, if the Pipeline is deleted, all the associated entities like lineage, - * etc., with that Pipeline will be deleted - */ - markDeletedPipelines?: boolean; - /** - * Regex exclude pipelines. - */ - pipelineFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete MlModels in OpenMetadata if the source MlModels are - * deleted. Also, if the MlModel is deleted, all the associated entities like lineage, etc., - * with that MlModels will be deleted - */ - markDeletedMlModels?: boolean; - /** - * Regex to only fetch MlModels with names matching the pattern. - */ - mlModelFilterPattern?: FilterPattern; - /** - * Regex to only fetch containers that matches the pattern. - */ - containerFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete containers in OpenMetadata if the source containers - * are deleted. Also, if the topic is deleted, all the associated entities with that - * containers will be deleted - */ - markDeletedContainers?: boolean; - storageMetadataConfigSource?: StorageMetadataConfigurationSource; - /** - * Regex to only include/exclude directories that matches the pattern. - */ - directoryFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude files that matches the pattern. - */ - fileFilterPattern?: FilterPattern; - /** - * Optional configuration to turn off fetching metadata for directories. - */ - includeDirectories?: boolean; - /** - * Optional configuration to turn off fetching metadata for files. - */ - includeFiles?: boolean; - /** - * Optional configuration to turn off fetching metadata for spreadsheets. - */ - includeSpreadsheets?: boolean; - /** - * Optional configuration to turn off fetching metadata for worksheets. - */ - includeWorksheets?: boolean; - /** - * Optional configuration to soft delete directories in OpenMetadata if the source - * directories are deleted. Also, if the directory is deleted, all the associated entities - * like files, spreadsheets, worksheets, lineage, etc., with that directory will be deleted - */ - markDeletedDirectories?: boolean; - /** - * Optional configuration to soft delete files in OpenMetadata if the source files are - * deleted. Also, if the file is deleted, all the associated entities like lineage, etc., - * with that file will be deleted - */ - markDeletedFiles?: boolean; - /** - * Optional configuration to soft delete spreadsheets in OpenMetadata if the source - * spreadsheets are deleted. Also, if the spreadsheet is deleted, all the associated - * entities like worksheets, lineage, etc., with that spreadsheet will be deleted - */ - markDeletedSpreadsheets?: boolean; - /** - * Optional configuration to soft delete worksheets in OpenMetadata if the source worksheets - * are deleted. Also, if the worksheet is deleted, all the associated entities like lineage, - * etc., with that worksheet will be deleted - */ - markDeletedWorksheets?: boolean; - /** - * Regex to only include/exclude spreadsheets that matches the pattern. - */ - spreadsheetFilterPattern?: FilterPattern; - /** - * Regex to only include/exclude worksheets that matches the pattern. - */ - worksheetFilterPattern?: FilterPattern; - /** - * Enable the 'Include Index Template' toggle to manage the ingestion of index template data. - */ - includeIndexTemplate?: boolean; - /** - * Optional configuration to turn off fetching sample data for search index. - */ - includeSampleData?: boolean; - /** - * Optional configuration to soft delete search indexes in OpenMetadata if the source search - * indexes are deleted. Also, if the search index is deleted, all the associated entities - * like lineage, etc., with that search index will be deleted - */ - markDeletedSearchIndexes?: boolean; - /** - * No. of records of sample data we want to ingest. - */ - sampleSize?: number; - /** - * Regex to only fetch search indexes that matches the pattern. - */ - searchIndexFilterPattern?: FilterPattern; - /** - * Fully qualified name of the entity to be tested, if we're working with a basic suite. - */ - entityFullyQualifiedName?: string; - /** - * Service connections to be used for the logical test suite. - */ - serviceConnections?: ServiceConnections[]; - /** - * List of test cases to be executed on the entity. If null, all test cases will be executed. - */ - testCases?: string[]; - /** - * Maximum number of events entities in a batch (Default 1000). - */ - batchSize?: number; - /** - * Certificate path to be added in configuration. The path should be local in the Ingestion - * Container. - */ - caCerts?: string; - recreateIndex?: boolean; - /** - * Region name. Required when using AWS Credentials. - */ - regionName?: string; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Connection Timeout - */ - timeout?: number; - /** - * Indicates whether to use aws credentials when connecting to OpenSearch in AWS. - */ - useAwsCredentials?: boolean; - /** - * Indicates whether to use SSL when connecting to ElasticSearch. By default, we will ignore - * SSL settings. - */ - useSSL?: boolean; - /** - * Indicates whether to verify certificates when using SSL connection to ElasticSearch. - * Ignored by default. Is set to true, make sure to send the certificates in the property - * `CA Certificates`. - */ - verifyCerts?: boolean; - /** - * Custom OpenMetadata Classification name for dbt tags. - */ - dbtClassificationName?: string; - /** - * Available sources to fetch DBT catalog and manifest files. - */ - dbtConfigSource?: DBTConfigurationSource; - /** - * Optional configuration to update the description from DBT or not - */ - dbtUpdateDescriptions?: boolean; - /** - * Optional configuration to update the owners from DBT or not - */ - dbtUpdateOwners?: boolean; - /** - * Optional configuration to search across databases for tables or not - */ - searchAcrossDatabases?: boolean; - /** - * Regex to only fetch tags that matches the pattern. - */ - tagFilterPattern?: FilterPattern; - /** - * Application configuration - */ - appConfig?: any[] | boolean | number | null | CollateAIAppConfig | string; - /** - * Application private configuration - */ - appPrivateConfig?: PrivateConfig; - /** - * Source Python Class Name to run the application - */ - sourcePythonClass?: string; - /** - * Regex to only fetch api collections with names matching the pattern. - */ - apiCollectionFilterPattern?: FilterPattern; - /** - * Optional configuration to soft delete api collections in OpenMetadata if the source - * collections are deleted. Also, if the collection is deleted, all the associated entities - * like endpoints, etc., with that collection will be deleted - */ - markDeletedApiCollections?: boolean; - /** - * Optional value of the ingestion runner name responsible for running the workflow - */ - ingestionRunner?: string; - /** - * List of operations to be performed on the service - */ - operations?: Operation[]; - /** - * Service to be modified - */ - service?: EntityReference; -} - -/** - * Configuration for the CollateAI External Application. - * - * Configuration for the Automator External Application. - * - * This schema defines the Slack App Token Configuration - * - * No configuration needed to instantiate the Data Insights Pipeline. The logic is handled - * in the backend. - * - * Search Indexing App. - * - * Configuration for the Collate AI Quality Agent. - * - * Configuration for the AutoPilot Application. - */ -export interface CollateAIAppConfig { - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - filter?: string; - /** - * Patch the description if it is empty, instead of raising a suggestion - */ - patchIfEmpty?: boolean; - /** - * Application Type - */ - type?: CollateAIAppConfigType; - /** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - */ - actions?: Action[]; - /** - * Entities selected to run the automation. - */ - resources?: Resource; - /** - * Bot Token - */ - botToken?: string; - /** - * User Token - */ - userToken?: string; - backfillConfiguration?: BackfillConfiguration; - /** - * Maximum number of events processed at a time (Default 100). - * - * Maximum number of events sent in a batch (Default 100). - */ - batchSize?: number; - moduleConfiguration?: ModuleConfiguration; - /** - * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property - * Type and are facing errors. Bear in mind that recreating the index will delete your - * DataAssets and a backfill will be needed. - */ - recreateDataAssetsIndex?: boolean; - sendToAdmins?: boolean; - sendToTeams?: boolean; - /** - * Enable automatic performance tuning based on cluster capabilities and database entity - * count - */ - autoTune?: boolean; - /** - * Number of threads to use for reindexing - */ - consumerThreads?: number; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Initial backoff time in milliseconds - */ - initialBackoff?: number; - /** - * Maximum backoff time in milliseconds - */ - maxBackoff?: number; - /** - * Maximum number of concurrent requests to the search index - */ - maxConcurrentRequests?: number; - /** - * Maximum number of retries for a failed request - */ - maxRetries?: number; - /** - * Maximum number of events sent in a batch (Default 100). - */ - payLoadSize?: number; - /** - * Number of threads to use for reindexing - */ - producerThreads?: number; - /** - * Queue Size to user internally for reindexing. - */ - queueSize?: number; - /** - * This schema publisher run modes. - */ - recreateIndex?: boolean; - /** - * Recreate Indexes with updated Language - */ - searchIndexMappingLanguage?: SearchIndexMappingLanguage; - /** - * Whether the suggested tests should be active or not upon suggestion - * - * Whether the AutoPilot Workflow should be active or not. - */ - active?: boolean; - /** - * Enter the retention period for Activity Threads of type = 'Conversation' records in days - * (e.g., 30 for one month, 60 for two months). - */ - activityThreadsRetentionPeriod?: number; - /** - * Enter the retention period for change event records in days (e.g., 7 for one week, 30 for - * one month). - */ - changeEventRetentionPeriod?: number; - /** - * Service Entity Link for which to trigger the application. - */ - entityLink?: string; - [property: string]: any; -} - -/** - * Action to take on those entities. E.g., propagate description through lineage, auto - * tagging, etc. - * - * Apply Classification Tags to the selected assets. - * - * Remove Classification Tags Action Type - * - * Apply Glossary Terms to the selected assets. - * - * Remove Glossary Terms Action Type - * - * Add domains to the selected assets. - * - * Remove domains from the selected assets. - * - * Apply Tags to the selected assets. - * - * Add a Custom Property to the selected assets. - * - * Remove Owner Action Type - * - * Add an owner to the selected assets. - * - * Add Test Cases to the selected assets. - * - * Remove Test Cases Action Type - * - * Add owners to the selected assets. - * - * Remove Custom Properties Action Type - * - * Add a Data Product to the selected assets. - * - * Remove a Data Product to the selected assets. - * - * Propagate description, tags and glossary terms via lineage - * - * ML Tagging action configuration for external automator. - */ -export interface Action { - /** - * Apply tags to the children of the selected assets that match the criteria. E.g., columns, - * tasks, topic fields,... - * - * Remove tags from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply terms to the children of the selected assets that match the criteria. E.g., - * columns, tasks, topic fields,... - * - * Remove terms from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Apply the description to the children of the selected assets that match the criteria. - * E.g., columns, tasks, topic fields,... - * - * Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic - * fields,... - * - * Add tests to the selected table columns - * - * Remove tests to the selected table columns - */ - applyToChildren?: string[]; - /** - * Update tags even if they are already defined in the asset. By default, incoming tags are - * merged with the existing ones. - * - * Update terms even if they are already defined in the asset. By default, incoming terms - * are merged with the existing ones. - * - * Update the domains even if they are defined in the asset. By default, we will only apply - * the domains to assets without domains. - * - * Update the description even if they are already defined in the asset. By default, we'll - * only add the descriptions to assets without the description set. - * - * Update the Custom Property even if it is defined in the asset. By default, we will only - * apply the owners to assets without the given Custom Property informed. - * - * Update the tier even if it is defined in the asset. By default, we will only apply the - * tier to assets without tier. - * - * Update the test even if it is defined in the asset. By default, we will only apply the - * test to assets without the existing test already existing. - * - * Update the owners even if it is defined in the asset. By default, we will only apply the - * owners to assets without owner. - * - * Update the Data Product even if the asset belongs to a different Domain. By default, we - * will only add the Data Product if the asset has no Domain, or it belongs to the same - * domain as the Data Product. - * - * Update descriptions, tags and Glossary Terms via lineage even if they are already defined - * in the asset. By default, descriptions are only updated if they are not already defined - * in the asset, and incoming tags are merged with the existing ones. - */ - overwriteMetadata?: boolean; - /** - * Classification Tags to apply (source must be 'Classification') - * - * Classification Tags to remove (source must be 'Classification') - */ - tags?: TierElement[]; - /** - * Application Type - */ - type: ActionType; - /** - * Remove tags from all the children and parent of the selected assets. - * - * Remove terms from all the children and parent of the selected assets. - * - * Remove descriptions from all the children and parent of the selected assets. - */ - applyToAll?: boolean; - /** - * Remove tags by its label type - * - * Remove terms by its label type - */ - labels?: LabelElement[]; - /** - * Glossary Terms to apply - * - * Glossary Terms to remove - */ - terms?: TierElement[]; - /** - * Domains to apply - */ - domains?: EntityReference[]; - /** - * Description to apply - */ - description?: string; - /** - * Owners to apply - * - * Custom Properties keys to remove - */ - customProperties?: any; - /** - * tier to apply - */ - tier?: TierElement; - /** - * Test Cases to apply - */ - testCases?: TestCaseDefinitions[]; - /** - * Remove all test cases - */ - removeAll?: boolean; - /** - * Test Cases to remove - */ - testCaseDefinitions?: string[]; - /** - * Owners to apply - */ - owners?: EntityReference[]; - /** - * Data Products to apply - * - * Data Products to remove - */ - dataProducts?: EntityReference[]; - /** - * Propagate the metadata to columns via column-level lineage. - */ - propagateColumnLevel?: boolean; - /** - * Propagate description through lineage - */ - propagateDescription?: boolean; - /** - * Propagate domains from the parent through lineage - */ - propagateDomains?: boolean; - /** - * Propagate glossary terms through lineage - */ - propagateGlossaryTerms?: boolean; - /** - * Propagate owner from the parent - */ - propagateOwner?: boolean; - /** - * Propagate the metadata to the parents (e.g., tables) via lineage. - */ - propagateParent?: boolean; - /** - * Propagate tags through lineage - */ - propagateTags?: boolean; - /** - * Propagate tier from the parent - */ - propagateTier?: boolean; - /** - * Number of levels to propagate lineage. If not set, it will propagate to all levels. - */ - propagationDepth?: number; - /** - * List of configurations to stop propagation based on conditions - */ - propagationStopConfigs?: PropagationStopConfig[]; -} - -/** - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Service to be modified - */ -export interface EntityReference { - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Link to the entity resource. - */ - href?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Name of the entity instance. - */ - name?: string; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type: string; -} - -/** - * Remove tags by its label type - * - * Remove terms by its label type - */ -export enum LabelElement { - Automated = "Automated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Configuration to stop lineage propagation based on conditions - */ -export interface PropagationStopConfig { - /** - * The metadata attribute to check for stopping propagation - */ - metadataAttribute: MetadataAttribute; - /** - * List of attribute values that will stop propagation when any of them is matched - */ - value: Array; -} - -/** - * The metadata attribute to check for stopping propagation - */ -export enum MetadataAttribute { - Description = "description", - Domains = "domains", - GlossaryTerms = "glossaryTerms", - Owner = "owner", - Tags = "tags", - Tier = "tier", -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - * - * Domains to apply - * - * This schema defines the EntityReferenceList type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * This schema defines the EntityReference type used for referencing an entity. - * EntityReference is used for capturing relationships from one entity to another. For - * example, a table has an attribute called database of type EntityReference that captures - * the relationship of a table `belongs to a` database. - * - * Service to be modified - */ -export interface TagLabel { - /** - * Description for the tag label. - * - * Optional description of entity. - */ - description?: string; - /** - * Display Name that identifies this tag. - * - * Display Name that identifies this entity. - */ - displayName?: string; - /** - * Link to the tag resource. - * - * Link to the entity resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType?: LabelTypeEnum; - /** - * Name of the tag or glossary term. - * - * Name of the entity instance. - */ - name?: string; - /** - * Label is from Tags or Glossary. - */ - source?: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state?: State; - style?: Style; - tagFQN?: string; - /** - * If true the entity referred to has been soft-deleted. - */ - deleted?: boolean; - /** - * Fully qualified name of the entity instance. For entities such as tables, databases - * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy - * such as `user` and `team` this will be same as the `name` field. - */ - fullyQualifiedName?: string; - /** - * Unique identifier that identifies an entity instance. - */ - id?: string; - /** - * If true the relationship indicated by this entity reference is inherited from the parent - * entity. - */ - inherited?: boolean; - /** - * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, - * `dashboardService`... - */ - type?: string; -} - -/** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ -export enum LabelTypeEnum { - Automated = "Automated", - Derived = "Derived", - Generated = "Generated", - Manual = "Manual", - Propagated = "Propagated", -} - -/** - * Label is from Tags or Glossary. - */ -export enum TagSource { - Classification = "Classification", - Glossary = "Glossary", -} - -/** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ -export enum State { - Confirmed = "Confirmed", - Suggested = "Suggested", -} - -/** - * UI Style is used to associate a color code and/or icon to entity to customize the look of - * that entity in UI. - */ -export interface Style { - /** - * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. - */ - color?: string; - /** - * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. - */ - iconURL?: string; -} - -/** - * This schema defines the type for labeling an entity with a Tag. - * - * tier to apply - */ -export interface TierElement { - /** - * Description for the tag label. - */ - description?: string; - /** - * Display Name that identifies this tag. - */ - displayName?: string; - /** - * Link to the tag resource. - */ - href?: string; - /** - * Label type describes how a tag label was applied. 'Manual' indicates the tag label was - * applied by a person. 'Derived' indicates a tag label was derived using the associated tag - * relationship (see Classification.json for more details). 'Propagated` indicates a tag - * label was propagated from upstream based on lineage. 'Automated' is used when a tool was - * used to determine the tag label. - */ - labelType: LabelTypeEnum; - /** - * Name of the tag or glossary term. - */ - name?: string; - /** - * Label is from Tags or Glossary. - */ - source: TagSource; - /** - * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the - * entity must confirm the suggested labels before it is marked as 'Confirmed'. - */ - state: State; - style?: Style; - tagFQN: string; -} - -/** - * Minimum set of requirements to get a Test Case request ready - */ -export interface TestCaseDefinitions { - /** - * Compute the passed and failed row count for the test case. - */ - computePassedFailedRowCount?: boolean; - parameterValues?: TestCaseParameterValue[]; - /** - * Tags to apply - */ - tags?: TierElement[]; - /** - * Fully qualified name of the test definition. - */ - testDefinition?: string; - /** - * If the test definition supports it, use dynamic assertion to evaluate the test case. - */ - useDynamicAssertion?: boolean; - [property: string]: any; -} - -/** - * This schema defines the parameter values that can be passed for a Test Case. - */ -export interface TestCaseParameterValue { - /** - * name of the parameter. Must match the parameter names in testCaseParameterDefinition - */ - name?: string; - /** - * value to be passed for the Parameters. These are input from Users. We capture this in - * string and convert during the runtime. - */ - value?: string; - [property: string]: any; -} - -/** - * Application Type - * - * Add Tags action type. - * - * Remove Classification Tags Action Type. - * - * Add Terms action type. - * - * Remove Terms Action Type. - * - * Add Domain Action Type. - * - * Remove Domain Action Type - * - * Add Description Action Type. - * - * Add Custom Properties Action Type. - * - * Remove Description Action Type - * - * Add Tier Action Type. - * - * Remove Tier Action Type - * - * Add Test Case Action Type. - * - * Remove Test Case Action Type - * - * Add Owner Action Type. - * - * Remove Owner Action Type - * - * Remove Custom Properties Action Type. - * - * Add Data Products Action Type. - * - * Remove Data Products Action Type. - * - * Lineage propagation action type. - * - * ML PII Tagging action type. - */ -export enum ActionType { - AddCustomPropertiesAction = "AddCustomPropertiesAction", - AddDataProductAction = "AddDataProductAction", - AddDescriptionAction = "AddDescriptionAction", - AddDomainAction = "AddDomainAction", - AddOwnerAction = "AddOwnerAction", - AddTagsAction = "AddTagsAction", - AddTermsAction = "AddTermsAction", - AddTestCaseAction = "AddTestCaseAction", - AddTierAction = "AddTierAction", - LineagePropagationAction = "LineagePropagationAction", - MLTaggingAction = "MLTaggingAction", - RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", - RemoveDataProductAction = "RemoveDataProductAction", - RemoveDescriptionAction = "RemoveDescriptionAction", - RemoveDomainAction = "RemoveDomainAction", - RemoveOwnerAction = "RemoveOwnerAction", - RemoveTagsAction = "RemoveTagsAction", - RemoveTermsAction = "RemoveTermsAction", - RemoveTestCaseAction = "RemoveTestCaseAction", - RemoveTierAction = "RemoveTierAction", -} - -/** - * Backfill Configuration - */ -export interface BackfillConfiguration { - /** - * Enable Backfill for the configured dates - */ - enabled?: boolean; - /** - * Date for which the backfill will end - */ - endDate?: Date; - /** - * Date from which to start the backfill - */ - startDate?: Date; - [property: string]: any; -} - -/** - * Different Module Configurations - */ -export interface ModuleConfiguration { - /** - * App Analytics Module configuration - */ - appAnalytics: AppAnalyticsConfig; - /** - * Cost Analysis Insights Module configuration - */ - costAnalysis: CostAnalysisConfig; - /** - * Data Assets Insights Module configuration - */ - dataAssets: DataAssetsConfig; - /** - * Data Quality Insights Module configuration - */ - dataQuality: DataQualityConfig; -} - -/** - * App Analytics Module configuration - */ -export interface AppAnalyticsConfig { - /** - * If Enabled, App Analytics insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Cost Analysis Insights Module configuration - */ -export interface CostAnalysisConfig { - /** - * If Enabled, Cost Analysis insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Data Assets Insights Module configuration - */ -export interface DataAssetsConfig { - /** - * If Enabled, Data Asset insights will be populated when the App runs. - */ - enabled: boolean; - /** - * List of Entities to Reindex - */ - entities?: string[]; - /** - * Defines the number of days the Data Assets Insights information will be kept. After it - * they will be deleted. - */ - retention?: number; - serviceFilter?: ServiceFilter; -} - -export interface ServiceFilter { - serviceName?: string; - serviceType?: string; -} - -/** - * Data Quality Insights Module configuration - */ -export interface DataQualityConfig { - /** - * If Enabled, Data Quality insights will be populated when the App runs. - */ - enabled: boolean; -} - -/** - * Entities selected to run the automation. - */ -export interface Resource { - /** - * Filter JSON tree to be used for rendering the filters in the UI. This comes from - * Immutable Tree type of react-awesome-query-builder. - */ - filterJsonTree?: string; - /** - * Query filter to be passed to ES. E.g., - * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG - * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. - */ - queryFilter?: string; - /** - * Type of the entity. E.g., 'table', 'chart',... - */ - type?: string[]; - [property: string]: any; -} - -/** - * Application Type - * - * Application type. - */ -export enum CollateAIAppConfigType { - AutoPilotApplication = "AutoPilotApplication", - Automator = "Automator", - CollateAI = "CollateAI", - CollateAIQualityAgent = "CollateAIQualityAgent", - DataInsights = "DataInsights", - DataInsightsReport = "DataInsightsReport", - SearchIndexing = "SearchIndexing", -} - -/** - * Application private configuration - * - * Private Configuration for the CollateAI External Application. - */ -export interface PrivateConfig { - /** - * Collate Server public URL. WAII will use this information to interact with the server. - * E.g., https://sandbox.getcollate.io - */ - collateURL?: string; - /** - * Limits for the CollateAI Application. - */ - limits?: AppLimitsConfig; - /** - * WAII API Token - */ - token?: string; - /** - * WAII API host URL - */ - waiiInstance?: string; - [property: string]: any; -} - -/** - * Limits for the CollateAI Application. - * - * Private Configuration for the App Limits. - */ -export interface AppLimitsConfig { - /** - * The records of the limits. - */ - actions: { [key: string]: number }; - /** - * The start of this limit cycle. - */ - billingCycleStart: Date; -} - -/** - * Available sources to fetch DBT catalog and manifest files. - * - * dbt Cloud configuration. - * - * DBT Catalog, Manifest and Run Results file path config. - * - * DBT Catalog, Manifest and Run Results HTTP path configuration. - * - * DBT Catalog, Manifest and Run Results files in S3 bucket. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in GCS storage. We will search for - * catalog.json, manifest.json and run_results.json. - * - * DBT Catalog, Manifest and Run Results files in Azure bucket. We will search for - * catalog.json, manifest.json and run_results.json. - */ -export interface DBTConfigurationSource { - /** - * dbt cloud account Id - */ - dbtCloudAccountId?: string; - /** - * dbt cloud account authentication token - */ - dbtCloudAuthToken?: string; - /** - * dbt cloud job id. - */ - dbtCloudJobId?: string; - /** - * In case of multiple projects in a dbt cloud account, specify the project's id from which - * you want to extract the dbt run artifacts - */ - dbtCloudProjectId?: string; - /** - * URL to connect to your dbt cloud instance. E.g., https://cloud.getdbt.com or - * https://emea.dbt.com/ - */ - dbtCloudUrl?: string; - /** - * dbt Configuration type - */ - dbtConfigType: DbtConfigType; - /** - * DBT catalog file path to extract dbt models with their column schemas. - */ - dbtCatalogFilePath?: string; - /** - * DBT manifest file path to extract dbt models and associate with tables. - */ - dbtManifestFilePath?: string; - /** - * DBT run results file path to extract the test results information. - */ - dbtRunResultsFilePath?: string; - /** - * DBT sources file path to extract the freshness test result. - */ - dbtSourcesFilePath?: string; - /** - * DBT catalog http file path to extract dbt models with their column schemas. - */ - dbtCatalogHttpPath?: string; - /** - * DBT manifest http file path to extract dbt models and associate with tables. - */ - dbtManifestHttpPath?: string; - /** - * DBT run results http file path to extract the test results information. - */ - dbtRunResultsHttpPath?: string; - /** - * DBT sources http file path to extract freshness test results information. - */ - dbtSourcesHttpPath?: string; - /** - * Details of the bucket where the dbt files are stored - */ - dbtPrefixConfig?: DBTPrefixConfig; - dbtSecurityConfig?: SecurityConfigClass; -} - -/** - * dbt Configuration type - */ -export enum DbtConfigType { - Azure = "azure", - Cloud = "cloud", - Gcs = "gcs", - HTTP = "http", - Local = "local", - S3 = "s3", -} - -/** - * Details of the bucket where the dbt files are stored - */ -export interface DBTPrefixConfig { - /** - * Name of the bucket where the dbt files are stored - */ - dbtBucketName?: string; - /** - * Path of the folder where the dbt files are stored - */ - dbtObjectPrefix?: string; -} - -/** - * Use incremental Metadata extraction after the first execution. This is commonly done by - * getting the changes from Audit tables on the supporting databases. - */ -export interface IncrementalMetadataExtractionConfiguration { - /** - * If True, enables Metadata Extraction to be incremental - */ - enabled: boolean; - /** - * Number os days to search back for a successful pipeline run. The timestamp of the last - * found successful pipeline run will be used as a base to search for updated entities. - */ - lookbackDays?: number; - /** - * Number of days to add to the last successful pipeline run timestamp to search for updated - * entities. - */ - safetyMarginDays?: number; -} - -/** - * Details required to generate Lineage - */ -export interface LineageInformation { - /** - * List of service path prefixes for lineage matching. Supported formats: DBServiceName, - * DBServiceName.DatabaseName, DBServiceName.DatabaseName.SchemaName, or - * DBServiceName.DatabaseName.SchemaName.TableName - */ - dbServicePrefixes?: string[]; - /** - * List of Database Service Names for creation of lineage - */ - dbServiceNames?: string[]; - /** - * List of Storage Service Names for creation of lineage - */ - storageServiceNames?: string[]; - [property: string]: any; -} - -/** - * Operation to be performed on the entity - */ -export interface Operation { - /** - * Entity to be modified - */ - entityLink: string; - /** - * The id of the operation - */ - id: string; - /** - * The configuration for the operation to be applied - */ - parameters: ReverseIngestionConfig; - /** - * Templated SQL command to be used for the operation. Context parameters will be populated - * based on the event type. - */ - SQLTemplate?: string; - /** - * Type of operation to perform - */ - type: OperationType; -} - -/** - * The configuration for the operation to be applied - * - * Configuration for updating descriptions - * - * Configuration for updating owners - * - * Configuration for updating tags - */ -export interface ReverseIngestionConfig { - /** - * New description of the service - */ - newDescription?: string; - /** - * Previous description of the service - */ - previousDescription?: string; - /** - * Added owners to be applied - */ - addedOwners?: EntityReference[]; - /** - * Removed owners from the entity - */ - removedOwners?: EntityReference[]; - /** - * Added tags to be applied - */ - addedTags?: TierElement[]; - /** - * Removed tags of the entity - */ - removedTags?: TierElement[]; -} - -/** - * Type of operation to perform - */ -export enum OperationType { - UpdateDescription = "UPDATE_DESCRIPTION", - UpdateOwner = "UPDATE_OWNER", - UpdateTags = "UPDATE_TAGS", -} - -/** - * Processing Engine Configuration. If not provided, the Native Engine will be used by - * default. - * - * Configuration for the native metadata ingestion engine - * - * This schema defines the configuration for a Spark Engine runner. - */ -export interface ProcessingEngine { - /** - * The type of the engine configuration - */ - type: ProcessingEngineType; - config?: Config; - /** - * Spark Connect Remote URL. - */ - remote?: string; -} - -export interface Config { - /** - * Additional Spark configuration properties as key-value pairs. - */ - extraConfig?: { [key: string]: any }; - /** - * Temporary path to store the data. - */ - tempPath?: string; - [property: string]: any; -} - -/** - * The type of the engine configuration - */ -export enum ProcessingEngineType { - Native = "Native", - Spark = "Spark", -} - -/** - * Type of Profile Sample (percentage or rows) - */ -export enum ProfileSampleType { - Percentage = "PERCENTAGE", - Rows = "ROWS", -} - -/** - * Type of Sampling Method (BERNOULLI or SYSTEM) - */ -export enum SamplingMethodType { - Bernoulli = "BERNOULLI", - System = "SYSTEM", -} - -/** - * Service connections available for the logical test suite. - */ -export interface ServiceConnections { - /** - * Connection configuration for the source. ex: mysql , tableau connection. - */ - serviceConnection: ServiceConnection; - serviceName: string; -} - -/** - * No manifest file available. Ingestion would look for bucket-level metadata file instead - * - * Storage Metadata Manifest file path config. - * - * Storage Metadata Manifest file HTTP path config. - * - * Storage Metadata Manifest file S3 path config. - * - * Storage Metadata Manifest file ADLS path config. - * - * Storage Metadata Manifest file GCS path config. - */ -export interface StorageMetadataConfigurationSource { - /** - * Storage Metadata manifest file path to extract locations to ingest from. - */ - manifestFilePath?: string; - /** - * Storage Metadata manifest http file path to extract locations to ingest from. - */ - manifestHttpPath?: string; - prefixConfig?: StorageMetadataBucketDetails; - securityConfig?: SecurityConfigClass; -} - -/** - * Details of the bucket where the storage metadata manifest file is stored - */ -export interface StorageMetadataBucketDetails { - /** - * Name of the top level container where the storage metadata file is stored - */ - containerName: string; - /** - * Path of the folder where the storage metadata file is stored. If the file is at the root, - * you can keep it empty. - */ - objectPrefix?: string; -} - -/** - * Pipeline type - * - * Database Source Config Metadata Pipeline type - * - * Database Source Config Usage Pipeline type - * - * Dashboard Source Config Metadata Pipeline type - * - * Messaging Source Config Metadata Pipeline type - * - * Profiler Source Config Pipeline type - * - * Pipeline Source Config Metadata Pipeline type - * - * MlModel Source Config Metadata Pipeline type - * - * Object Store Source Config Metadata Pipeline type - * - * Drive Source Config Metadata Pipeline type - * - * Search Source Config Metadata Pipeline type - * - * DBT Config Pipeline type - * - * Pipeline Source Config For Application Pipeline type. Nothing is required. - * - * Api Source Config Metadata Pipeline type - * - * Reverse Ingestion Config Pipeline type - */ -export enum FluffyType { - APIMetadata = "ApiMetadata", - Application = "Application", - AutoClassification = "AutoClassification", - DashboardMetadata = "DashboardMetadata", - DataInsight = "dataInsight", - DatabaseLineage = "DatabaseLineage", - DatabaseMetadata = "DatabaseMetadata", - DatabaseUsage = "DatabaseUsage", - Dbt = "DBT", - DriveMetadata = "DriveMetadata", - MessagingMetadata = "MessagingMetadata", - MetadataToElasticSearch = "MetadataToElasticSearch", - MlModelMetadata = "MlModelMetadata", - PipelineMetadata = "PipelineMetadata", - Profiler = "Profiler", - ReverseIngestion = "ReverseIngestion", - SearchMetadata = "SearchMetadata", - StorageMetadata = "StorageMetadata", - TestSuite = "TestSuite", -} - -/** - * Configuration for Stage Component in the OpenMetadata Ingestion Framework. - */ -export interface Stage { - config?: { [key: string]: any }; - /** - * Type of stage component ex: table-usage - */ - type: string; -} - -/** - * Configuration for the entire Ingestion Workflow. - */ -export interface WorkflowConfig { - config?: { [key: string]: any }; - loggerLevel?: LogLevels; - openMetadataServerConfig: OpenMetadataConnection; - /** - * Control if we want to flag the workflow as failed if we encounter any processing errors. - */ - raiseOnError?: boolean; - /** - * The percentage of successfully processed records that must be achieved for the pipeline - * to be considered successful. Otherwise, the pipeline will be marked as failed. - */ - successThreshold?: number; -} - -/** - * Supported logging levels - */ -export enum LogLevels { - Debug = "DEBUG", - Error = "ERROR", - Info = "INFO", - Warn = "WARN", -} - -/** - * OpenMetadata Connection Config - */ -export interface OpenMetadataConnection { - /** - * OpenMetadata server API version to use. - */ - apiVersion?: string; - /** - * OpenMetadata Server Authentication Provider. - */ - authProvider?: AuthProvider; - /** - * Cluster name to differentiate OpenMetadata Server instance - */ - clusterName?: string; - /** - * Regex to only include/exclude databases that matches the pattern. - */ - databaseFilterPattern?: FilterPattern; - /** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ - elasticsSearch?: OpenMetadataServerConfigElasticsSearch; - /** - * Validate Openmetadata Server & Client Version. - */ - enableVersionValidation?: boolean; - extraHeaders?: { [key: string]: string }; - /** - * Force the overwriting of any entity during the ingestion. - */ - forceEntityOverwriting?: boolean; - /** - * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api - */ - hostPort: string; - /** - * Include Dashboards for Indexing - */ - includeDashboards?: boolean; - /** - * Include Database Services for Indexing - */ - includeDatabaseServices?: boolean; - /** - * Include Glossary Terms for Indexing - */ - includeGlossaryTerms?: boolean; - /** - * Include Messaging Services for Indexing - */ - includeMessagingServices?: boolean; - /** - * Include MlModels for Indexing - */ - includeMlModels?: boolean; - /** - * Include Pipelines for Indexing - */ - includePipelines?: boolean; - /** - * Include Pipeline Services for Indexing - */ - includePipelineServices?: boolean; - /** - * Include Tags for Policy - */ - includePolicy?: boolean; - /** - * Include Tables for Indexing - */ - includeTables?: boolean; - /** - * Include Tags for Indexing - */ - includeTags?: boolean; - /** - * Include Teams for Indexing - */ - includeTeams?: boolean; - /** - * Include Topics for Indexing - */ - includeTopics?: boolean; - /** - * Include Users for Indexing - */ - includeUsers?: boolean; - /** - * Limit the number of records for Indexing. - */ - limitRecords?: number; - /** - * Regex to only include/exclude schemas that matches the pattern. - */ - schemaFilterPattern?: FilterPattern; - /** - * Secrets Manager Loader for the Pipeline Service Client. - */ - secretsManagerLoader?: SecretsManagerClientLoader; - /** - * Secrets Manager Provider for OpenMetadata Server. - */ - secretsManagerProvider?: SecretsManagerProvider; - /** - * OpenMetadata Client security configuration. - */ - securityConfig?: OpenMetadataJWTClientConfig; - /** - * SSL Configuration for OpenMetadata Server - */ - sslConfig?: ConsumerConfigSSLClass; - /** - * If set to true, when creating a service during the ingestion we will store its Service - * Connection. Otherwise, the ingestion will create a bare service without connection - * details. - */ - storeServiceConnection?: boolean; - /** - * Flag to enable Data Insight Extraction - */ - supportsDataInsightExtraction?: boolean; - /** - * Flag to enable ElasticSearch Reindexing Extraction - */ - supportsElasticSearchReindexingExtraction?: boolean; - /** - * Regex to only include/exclude tables that matches the pattern. - */ - tableFilterPattern?: FilterPattern; - /** - * Service Type - */ - type?: OpenmetadataType; - /** - * Flag to verify SSL Certificate for OpenMetadata Server. - */ - verifySSL?: VerifySSL; -} - -/** - * Configuration for Sink Component in the OpenMetadata Ingestion Framework. - */ -export interface OpenMetadataServerConfigElasticsSearch { - config?: { [key: string]: any }; - /** - * Type of sink component ex: metadata - */ - type: string; -} - -/** - * Service Type - * - * OpenMetadata service type - */ -export enum OpenmetadataType { - OpenMetadata = "OpenMetadata", -}