From 2c539f912dc3132a873c53f5ef2d240ca26b65bf Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Mon, 8 Jan 2024 12:28:25 +0530 Subject: [PATCH] Added alation connector docs (#14614) --- .../connectors/metadata/alation/index.md | 54 +++++ .../connectors/metadata/alation/yaml.md | 190 ++++++++++++++++++ .../v1.2.x/connectors/metadata/index.md | 1 + openmetadata-docs/content/v1.2.x/menu.md | 4 + .../connectors/metadata/alation/index.md | 54 +++++ .../connectors/metadata/alation/yaml.md | 190 ++++++++++++++++++ .../connectors/metadata/index.md | 1 + .../content/v1.3.x-SNAPSHOT/menu.md | 6 +- 8 files changed, 499 insertions(+), 1 deletion(-) create mode 100644 openmetadata-docs/content/v1.2.x/connectors/metadata/alation/index.md create mode 100644 openmetadata-docs/content/v1.2.x/connectors/metadata/alation/yaml.md create mode 100644 openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/alation/index.md create mode 100644 openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/alation/yaml.md diff --git a/openmetadata-docs/content/v1.2.x/connectors/metadata/alation/index.md b/openmetadata-docs/content/v1.2.x/connectors/metadata/alation/index.md new file mode 100644 index 00000000000..aab19132cdd --- /dev/null +++ b/openmetadata-docs/content/v1.2.x/connectors/metadata/alation/index.md @@ -0,0 +1,54 @@ +--- +title: Alation +slug: /connectors/metadata/alation +--- + +# Alation (Collate Only) +Alation connector is supported to run only externally and not via OpenMetadata UI. +Check the following docs to run the Ingestion Framework in any orchestrator externally. + +{% tilesContainer %} +{% tile + title="Run Connectors from the OpenMetadata UI" + description="Learn how to manage your deployment to run connectors from the UI" + link="/deployment/ingestion/openmetadata" + / %} +{% tile + title="Run the Connector Externally" + description="Get the YAML to run the ingestion externally" + link="/connectors/metadata/alation/yaml" + / %} +{% tile + title="External Schedulers" + description="Get more information about running the Ingestion Framework Externally" + link="/deployment/ingestion" + / %} +{% /tilesContainer %} + +## Requirements + +Follow the official documentation to generate a API Access Token from [here](https://developer.alation.com/dev/docs/authentication-into-alation-apis) + +## Data Mapping and Assumptions + +Following entities are supported and will be mapped to the OpenMetadata entities as shown below. + +{% multiTablesWrapper %} + +| Alation Entity | OpenMetadata Entity | +| :----------------------------| :--------------------------- | +| Data Source (OCF and Native) | Database Service | +| Data Source (OCF and Native) | Database | +| Schema | Schema | +| Table | Table | +| Columns | Columns | +| Custom Fields | Custom Properties | +| Tags | Tags | + +{% /multiTablesWrapper %} + +- Since Alation does not have a concept of Service entity, the Data Sources (OCF and Native) will be mapped to Database Service and Database in OpenMetadata. Hence for each Data Source in Alation there will one Database Service and Database present in OpenMetadata. +- Custom fields will have a 1:1 mapping for all the entities except for Columns since OpenMetadata does not support custom properties for columns. +- Alation has two fields for descriptions i.e. `descriptions` and `comments`. These fields will be combined under one field `description` in OpenMetadata for all the entities. + + diff --git a/openmetadata-docs/content/v1.2.x/connectors/metadata/alation/yaml.md b/openmetadata-docs/content/v1.2.x/connectors/metadata/alation/yaml.md new file mode 100644 index 00000000000..be627fcd58c --- /dev/null +++ b/openmetadata-docs/content/v1.2.x/connectors/metadata/alation/yaml.md @@ -0,0 +1,190 @@ +--- +title: Run the Alation Connector Externally +slug: /connectors/metadata/alation/yaml +--- + +# Run the Alation Connector Externally (Collate Only) + +In this section, we provide guides and references to use the Alation connector. + +Configure and schedule Alation metadata and profiler workflows from the OpenMetadata UI: + +- [Requirements](#requirements) +- [Metadata Ingestion](#metadata-ingestion) + + +{% partial file="/v1.2/connectors/external-ingestion-deployment.md" /%} + +## Requirements + +Follow the official documentation to generate a API Access Token from [here](https://developer.alation.com/dev/docs/authentication-into-alation-apis) + +## Data Mapping and Assumptions + +Following entities are supported and will be mapped to the OpenMetadata entities as shown below. + +{% multiTablesWrapper %} + +| Alation Entity | OpenMetadata Entity | +| :----------------------------| :--------------------------- | +| Data Source (OCF and Native) | Database Service | +| Data Source (OCF and Native) | Database | +| Schema | Schema | +| Table | Table | +| Columns | Columns | +| Custom Fields | Custom Properties | +| Tags | Tags | + +{% /multiTablesWrapper %} + +- Since Alation does not have a concept of Service entity, the Data Sources (OCF and Native) will be mapped to Database Service and Database in OpenMetadata. Hence for each Data Source in Alation there will one Database Service and Database present in OpenMetadata. +- Custom fields will have a 1:1 mapping for all the entities except for Columns since OpenMetadata does not support custom properties for columns. +- Alation has two fields for descriptions i.e. `descriptions` and `comments`. These fields will be combined under one field `description` in OpenMetadata for all the entities. + + +### Python Requirements + +To run the Alation ingestion, you will need to install: + +```bash +pip3 install "openmetadata-ingestion[alation]" +``` + +## Metadata Ingestion + +All connectors are defined as JSON Schemas. +[Here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/metadata/alationConnection.json) +you can find the structure to create a connection to Alation. + +In order to create and run a Metadata Ingestion workflow, we will follow +the steps to create a YAML configuration able to connect to the source, +process the Entities if needed, and reach the OpenMetadata server. + +The workflow is modeled around the following +[JSON Schema](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json) + +### 1. Define the YAML Config + +{% codePreview %} + +{% codeInfoContainer %} + +#### Source Configuration - Service Connection + +{% codeInfo srNumber=12 %} + +**hostPort**: Host and port of the Alation instance. + +{% /codeInfo %} + +{% codeInfo srNumber=13 %} + +**authType**: Following authentication types are supported: +1. Basic Authentication: +We'll use the user credentials to generate the access token required to authenticate Alation APIs +- username: Username of the user. +- password: Password of the user. + +2. Access Token Authentication: +The access token created using the steps mentioned [here](https://developer.alation.com/dev/docs/authentication-into-alation-apis#create-via-ui) can directly be entered. We'll use that directly to authenticate the Alation APIs +- accessToken: Generated access token + +{% /codeInfo %} + +{% codeInfo srNumber=14 %} + +**projectName**: Project Name can be anything. e.g Prod or Demo. It will be used while creating the tokens. + +{% /codeInfo %} + +{% codeInfo srNumber=15 %} + +**paginationLimit**: Pagination limit used for Alation APIs pagination. By default is set to 10. + +{% /codeInfo %} + +{% codeInfo srNumber=16 %} + +**includeUndeployedDatasources**: Specifies if undeployed datasources should be included while ingesting. By default is set to `false`. + +{% /codeInfo %} + +{% codeInfo srNumber=17 %} + +**includeHiddenDatasources**: Specifies if hidden datasources should be included while ingesting. By default is set to `false`. + +{% /codeInfo %} + +{% codeInfo srNumber=18 %} + +**alationTagClassificationName**: Specify the classification name under which the tags from alation will be created in OpenMetadata. By default it is set to `alationTags`. + +{% /codeInfo %} + +#### Sink Configuration + +{% codeInfo srNumber=19 %} + +To send the metadata to OpenMetadata, it needs to be specified as `type: metadata-rest`. + +{% /codeInfo %} + +{% partial file="/v1.2/connectors/yaml/workflow-config-def.md" /%} +{% /codeInfoContainer %} + +{% codeBlock fileName="filename.yaml" %} + +```yaml +source: + type: Alation + serviceName: local_alation + serviceConnection: + config: + type: Alation +``` +```yaml {% srNumber=12 %} + hostPort: http://localhost:21000 +``` +```yaml {% srNumber=13 %} + # Select one authentication type from below + # For Basic Authentication + authType: + username: username + password: password + # # For Access Token Authentication + # authType: + # accessToken: api_access_token +``` +```yaml {% srNumber=14 %} + projectName: Test +``` +```yaml {% srNumber=15 %} + paginationLimit: 10 +``` +```yaml {% srNumber=16 %} + includeUndeployedDatasources: false # true or false +``` +```yaml {% srNumber=17 %} + includeHiddenDatasources: false # true or false +``` +```yaml {% srNumber=18 %} + alationTagClassificationName: alationTags +``` +```yaml + sourceConfig: + config: + type: DatabaseMetadata +``` +```yaml {% srNumber=19 %} +sink: + type: metadata-rest + config: {} +``` + +{% partial file="/v1.2/connectors/yaml/workflow-config.md" /%} + +{% /codeBlock %} + +{% /codePreview %} + +{% partial file="/v1.2/connectors/yaml/ingestion-cli.md" /%} diff --git a/openmetadata-docs/content/v1.2.x/connectors/metadata/index.md b/openmetadata-docs/content/v1.2.x/connectors/metadata/index.md index 47367584106..49381bc09d6 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/metadata/index.md +++ b/openmetadata-docs/content/v1.2.x/connectors/metadata/index.md @@ -9,6 +9,7 @@ This is the supported list of connectors for Metadata Services: - [Amundsen](/connectors/metadata/amundsen) - [Atlas](/connectors/metadata/atlas) +- [Alation](/connectors/metadata/alation) If you have a request for a new connector, don't hesitate to reach out in [Slack](https://slack.open-metadata.org/) or open a [feature request](https://github.com/open-metadata/OpenMetadata/issues/new/choose) in our GitHub repo. diff --git a/openmetadata-docs/content/v1.2.x/menu.md b/openmetadata-docs/content/v1.2.x/menu.md index 30f8c76fbd4..dc68ea77f51 100644 --- a/openmetadata-docs/content/v1.2.x/menu.md +++ b/openmetadata-docs/content/v1.2.x/menu.md @@ -675,6 +675,10 @@ site_menu: url: /connectors/metadata/atlas - category: Connectors / Metadata / Atlas / Run Externally url: /connectors/metadata/atlas/yaml + - category: Connectors / Metadata / Alation + url: /connectors/metadata/alation + - category: Connectors / Metadata / Alation / Run Externally + url: /connectors/metadata/alation/yaml - category: Connectors / Custom Connectors url: /connectors/custom-connectors diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/alation/index.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/alation/index.md new file mode 100644 index 00000000000..aab19132cdd --- /dev/null +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/alation/index.md @@ -0,0 +1,54 @@ +--- +title: Alation +slug: /connectors/metadata/alation +--- + +# Alation (Collate Only) +Alation connector is supported to run only externally and not via OpenMetadata UI. +Check the following docs to run the Ingestion Framework in any orchestrator externally. + +{% tilesContainer %} +{% tile + title="Run Connectors from the OpenMetadata UI" + description="Learn how to manage your deployment to run connectors from the UI" + link="/deployment/ingestion/openmetadata" + / %} +{% tile + title="Run the Connector Externally" + description="Get the YAML to run the ingestion externally" + link="/connectors/metadata/alation/yaml" + / %} +{% tile + title="External Schedulers" + description="Get more information about running the Ingestion Framework Externally" + link="/deployment/ingestion" + / %} +{% /tilesContainer %} + +## Requirements + +Follow the official documentation to generate a API Access Token from [here](https://developer.alation.com/dev/docs/authentication-into-alation-apis) + +## Data Mapping and Assumptions + +Following entities are supported and will be mapped to the OpenMetadata entities as shown below. + +{% multiTablesWrapper %} + +| Alation Entity | OpenMetadata Entity | +| :----------------------------| :--------------------------- | +| Data Source (OCF and Native) | Database Service | +| Data Source (OCF and Native) | Database | +| Schema | Schema | +| Table | Table | +| Columns | Columns | +| Custom Fields | Custom Properties | +| Tags | Tags | + +{% /multiTablesWrapper %} + +- Since Alation does not have a concept of Service entity, the Data Sources (OCF and Native) will be mapped to Database Service and Database in OpenMetadata. Hence for each Data Source in Alation there will one Database Service and Database present in OpenMetadata. +- Custom fields will have a 1:1 mapping for all the entities except for Columns since OpenMetadata does not support custom properties for columns. +- Alation has two fields for descriptions i.e. `descriptions` and `comments`. These fields will be combined under one field `description` in OpenMetadata for all the entities. + + diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/alation/yaml.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/alation/yaml.md new file mode 100644 index 00000000000..c621653bc9b --- /dev/null +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/alation/yaml.md @@ -0,0 +1,190 @@ +--- +title: Run the Alation Connector Externally +slug: /connectors/metadata/alation/yaml +--- + +# Run the Alation Connector Externally (Collate Only) + +In this section, we provide guides and references to use the Alation connector. + +Configure and schedule Alation metadata and profiler workflows from the OpenMetadata UI: + +- [Requirements](#requirements) +- [Metadata Ingestion](#metadata-ingestion) + + +{% partial file="/v1.3/connectors/external-ingestion-deployment.md" /%} + +## Requirements + +Follow the official documentation to generate a API Access Token from [here](https://developer.alation.com/dev/docs/authentication-into-alation-apis) + +## Data Mapping and Assumptions + +Following entities are supported and will be mapped to the OpenMetadata entities as shown below. + +{% multiTablesWrapper %} + +| Alation Entity | OpenMetadata Entity | +| :----------------------------| :--------------------------- | +| Data Source (OCF and Native) | Database Service | +| Data Source (OCF and Native) | Database | +| Schema | Schema | +| Table | Table | +| Columns | Columns | +| Custom Fields | Custom Properties | +| Tags | Tags | + +{% /multiTablesWrapper %} + +- Since Alation does not have a concept of Service entity, the Data Sources (OCF and Native) will be mapped to Database Service and Database in OpenMetadata. Hence for each Data Source in Alation there will one Database Service and Database present in OpenMetadata. +- Custom fields will have a 1:1 mapping for all the entities except for Columns since OpenMetadata does not support custom properties for columns. +- Alation has two fields for descriptions i.e. `descriptions` and `comments`. These fields will be combined under one field `description` in OpenMetadata for all the entities. + + +### Python Requirements + +To run the Alation ingestion, you will need to install: + +```bash +pip3 install "openmetadata-ingestion[alation]" +``` + +## Metadata Ingestion + +All connectors are defined as JSON Schemas. +[Here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/metadata/alationConnection.json) +you can find the structure to create a connection to Alation. + +In order to create and run a Metadata Ingestion workflow, we will follow +the steps to create a YAML configuration able to connect to the source, +process the Entities if needed, and reach the OpenMetadata server. + +The workflow is modeled around the following +[JSON Schema](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json) + +### 1. Define the YAML Config + +{% codePreview %} + +{% codeInfoContainer %} + +#### Source Configuration - Service Connection + +{% codeInfo srNumber=12 %} + +**hostPort**: Host and port of the Alation instance. + +{% /codeInfo %} + +{% codeInfo srNumber=13 %} + +**authType**: Following authentication types are supported: +1. Basic Authentication: +We'll use the user credentials to generate the access token required to authenticate Alation APIs +- username: Username of the user. +- password: Password of the user. + +2. Access Token Authentication: +The access token created using the steps mentioned [here](https://developer.alation.com/dev/docs/authentication-into-alation-apis#create-via-ui) can directly be entered. We'll use that directly to authenticate the Alation APIs +- accessToken: Generated access token + +{% /codeInfo %} + +{% codeInfo srNumber=14 %} + +**projectName**: Project Name can be anything. e.g Prod or Demo. It will be used while creating the tokens. + +{% /codeInfo %} + +{% codeInfo srNumber=15 %} + +**paginationLimit**: Pagination limit used for Alation APIs pagination. By default is set to 10. + +{% /codeInfo %} + +{% codeInfo srNumber=16 %} + +**includeUndeployedDatasources**: Specifies if undeployed datasources should be included while ingesting. By default is set to `false`. + +{% /codeInfo %} + +{% codeInfo srNumber=17 %} + +**includeHiddenDatasources**: Specifies if hidden datasources should be included while ingesting. By default is set to `false`. + +{% /codeInfo %} + +{% codeInfo srNumber=18 %} + +**alationTagClassificationName**: Specify the classification name under which the tags from alation will be created in OpenMetadata. By default it is set to `alationTags`. + +{% /codeInfo %} + +#### Sink Configuration + +{% codeInfo srNumber=19 %} + +To send the metadata to OpenMetadata, it needs to be specified as `type: metadata-rest`. + +{% /codeInfo %} + +{% partial file="/v1.3/connectors/yaml/workflow-config-def.md" /%} +{% /codeInfoContainer %} + +{% codeBlock fileName="filename.yaml" %} + +```yaml +source: + type: Alation + serviceName: local_alation + serviceConnection: + config: + type: Alation +``` +```yaml {% srNumber=12 %} + hostPort: http://localhost:21000 +``` +```yaml {% srNumber=13 %} + # Select one authentication type from below + # For Basic Authentication + authType: + username: username + password: password + # # For Access Token Authentication + # authType: + # accessToken: api_access_token +``` +```yaml {% srNumber=14 %} + projectName: Test +``` +```yaml {% srNumber=15 %} + paginationLimit: 10 +``` +```yaml {% srNumber=16 %} + includeUndeployedDatasources: false # true or false +``` +```yaml {% srNumber=17 %} + includeHiddenDatasources: false # true or false +``` +```yaml {% srNumber=18 %} + alationTagClassificationName: alationTags +``` +```yaml + sourceConfig: + config: + type: DatabaseMetadata +``` +```yaml {% srNumber=19 %} +sink: + type: metadata-rest + config: {} +``` + +{% partial file="/v1.3/connectors/yaml/workflow-config.md" /%} + +{% /codeBlock %} + +{% /codePreview %} + +{% partial file="/v1.3/connectors/yaml/ingestion-cli.md" /%} diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/index.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/index.md index 47367584106..49381bc09d6 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/index.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/metadata/index.md @@ -9,6 +9,7 @@ This is the supported list of connectors for Metadata Services: - [Amundsen](/connectors/metadata/amundsen) - [Atlas](/connectors/metadata/atlas) +- [Alation](/connectors/metadata/alation) If you have a request for a new connector, don't hesitate to reach out in [Slack](https://slack.open-metadata.org/) or open a [feature request](https://github.com/open-metadata/OpenMetadata/issues/new/choose) in our GitHub repo. diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md index 55dbb7307b9..9f2326159ab 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/menu.md @@ -676,7 +676,11 @@ site_menu: - category: Connectors / Metadata / Atlas url: /connectors/metadata/atlas - category: Connectors / Metadata / Atlas / Run Externally - url: /connectors/metadata/atlas/yaml + url: /connectors/metadata/ + - category: Connectors / Metadata / Alation + url: /connectors/metadata/alation + - category: Connectors / Metadata / Alation / Run Externally + url: /connectors/metadata/alation/yaml - category: Connectors / Custom Connectors url: /connectors/custom-connectors