diff --git a/openmetadata-docs/content/partials/v1.7/connectors/pipeline/connectors-list.md b/openmetadata-docs/content/partials/v1.7/connectors/pipeline/connectors-list.md index 822913ac440..9fe1377f308 100644 --- a/openmetadata-docs/content/partials/v1.7/connectors/pipeline/connectors-list.md +++ b/openmetadata-docs/content/partials/v1.7/connectors/pipeline/connectors-list.md @@ -16,5 +16,6 @@ {% connectorInfoCard name="NiFi" stage="PROD" href="/connectors/pipeline/nifi" platform="OpenMetadata" / %} {% connectorInfoCard name="Spline" stage="BETA" href="/connectors/pipeline/spline" platform="OpenMetadata" / %} {% connectorInfoCard name="Stitch" stage="PROD" href="/connectors/pipeline/stitch" platform="Collate" / %} +{% connectorInfoCard name="Wherescape" stage="BETA" href="/connectors/pipeline/wherescape" platform="Collate" / %} {% /connectorsListContainer %} \ No newline at end of file diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md index dddb8f6b21e..2264140d777 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md @@ -379,6 +379,10 @@ site_menu: url: /connectors/pipeline/airflow/mwaa - category: Connectors / Pipeline / Airflow / GCS Composer url: /connectors/pipeline/airflow/gcs-composer + - category: Connectors / Pipeline / Wherescape + url: /connectors/pipeline/wherescape + - category: Connectors / Pipeline / Wherescape / Run Externally + url: /connectors/pipeline/wherescape/yaml - category: Connectors / Pipeline / Dagster url: /connectors/pipeline/dagster - category: Connectors / Pipeline / Dagster / Run Externally diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/pipeline/wherescape/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/pipeline/wherescape/index.md new file mode 100644 index 00000000000..8b04a04d5af --- /dev/null +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/pipeline/wherescape/index.md @@ -0,0 +1,68 @@ +--- +title: Wherescape +slug: /connectors/pipeline/wherescape +--- + +{% connectorDetailsHeader +name="Wherescape" +stage="BETA" +platform="Collate" +availableFeatures=["Pipelines", "Pipeline Status", "Lineage", "Owners"] +unavailableFeatures=["Tags"] +/ %} + +In this section, we provide guides and references to use the Wherescape connector. + +Configure and schedule Wherescape metadata workflow from the OpenMetadata UI: + +- [Requirements](#requirements) +- [Metadata Ingestion](#metadata-ingestion) + - [Connection Details](#connection-details) + +{% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/pipeline/wherescape/yaml"} /%} + +## Requirements +To extract Wherescape metadata, we need the batabase connection details where the metadata is stored. + +- `API` Permission ( While Creating the User, from Admin -> User ) +- To retrieve lineage data, the user must be granted [Component-level permissions](https://docs.matillion.com/metl/docs/2932106/#component). +- To enable lineage tracking in Matillion, **Matillion Enterprise Mode** is required. For detailed setup instructions and further information, refer to the official documentation: [Matillion Lineage Documentation](https://docs.matillion.com/metl/docs/2881895/). + +## Metadata Ingestion + +{% partial + file="/v1.7/connectors/metadata-ingestion-ui.md" + variables={ + connector: "Wherescape", + selectServicePath: "/images/v1.7/connectors/wherescape/select-service.png", + addNewServicePath: "/images/v1.7/connectors/wherescape/add-new-service.png", + serviceConnectionPath: "/images/v1.7/connectors/wherescape/service-connection.png", +} +/%} + +{% stepsContainer %} +{% extraContent parentTagName="stepsContainer" %} + +#### Connection Details + +- **Connection**: Wherescape metadata database connection. + +In terms of `connection` we support the following selections: + +- **Microsoft SQL Server**: To connect to the Wherescape metadata database: + - Provide the SQL Server connection credentials including username and password + - Specify the database name where Wherescape metadata is stored + - Enter the host and port for the SQL Server instance + - The connector will establish a connection to this database to extract Wherescape pipeline metadata + +{% /extraContent %} + +{% partial file="/v1.7/connectors/test-connection.md" /%} + +{% partial file="/v1.7/connectors/pipeline/configure-ingestion.md" /%} + +{% partial file="/v1.7/connectors/ingestion-schedule-and-deploy.md" /%} + +{% /stepsContainer %} + +{% partial file="/v1.7/connectors/troubleshooting.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/pipeline/wherescape/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/pipeline/wherescape/yaml.md new file mode 100644 index 00000000000..1be52b6ed9d --- /dev/null +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/pipeline/wherescape/yaml.md @@ -0,0 +1,117 @@ +--- +title: Run the Wherescape Connector Externally +slug: /connectors/pipeline/wherescape/yaml +--- + +{% connectorDetailsHeader +name="Wherescape" +stage="BETA" +platform="Collate" +availableFeatures=["Pipelines", "Pipeline Status", "Lineage", "Owners"] +unavailableFeatures=["Tags"] +/ %} + +In this section, we provide guides and references to use the Wherescape connector. + +Configure and schedule Wherescape metadata workflows from the OpenMetadata UI: + +- [Requirements](#requirements) +- [Metadata Ingestion](#metadata-ingestion) + +{% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} + +## Requirements + +### Python Requirements + +{% partial file="/v1.7/connectors/python-requirements.md" /%} + +To run the Wherescape ingestion, you will need to install: + +```bash +pip3 install "openmetadata-ingestion[wherescape]" +``` + + +{% /note %} + + +## Metadata Ingestion + +All connectors are defined as JSON Schemas. +[Here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/wherescapeConnection.json) +you can find the structure to create a connection to Wherescape. + +In order to create and run a Metadata Ingestion workflow, we will follow +the steps to create a YAML configuration able to connect to the source, +process the Entities if needed, and reach the OpenMetadata server. + +The workflow is modeled around the following +[JSON Schema](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json) + +### 1. Define the YAML Config + +This is a sample config for Wherescape: + +{% codePreview %} + +{% codeInfoContainer %} + +#### Source Configuration - Service Connection + +{% codeInfo srNumber=1 %} + +{% /codeInfo %} + +{% codeInfo srNumber=1 %} + +**connection**: + +In terms of `connection` we support the following selections: + +- `Mssql`: Pass the required credentials to reach out this service. We will + create a connection to the pointed database and read Wherescape data from there. + +{% /codeInfo %} + +{% partial file="/v1.7/connectors/yaml/pipeline/source-config-def.md" /%} + +{% partial file="/v1.7/connectors/yaml/ingestion-sink-def.md" /%} + +{% partial file="/v1.7/connectors/yaml/workflow-config-def.md" /%} + +{% /codeInfoContainer %} + +{% codeBlock fileName="filename.yaml" %} + +```yaml {% isCodeBlock=true %} +source: + type: wherescape + serviceName: wherescape_source + serviceConnection: + config: + type: Wherescape +``` +```yaml {% srNumber=6 %} + # Connection needs to be Mssql + connection: + type: Mssql + username: user + authType: + password: pass + databaseSchema: db + hostPort: localhost:1433 +``` + +{% partial file="/v1.7/connectors/yaml/pipeline/source-config.md" /%} + +{% partial file="/v1.7/connectors/yaml/ingestion-sink.md" /%} + +{% partial file="/v1.7/connectors/yaml/workflow-config.md" /%} + +{% /codeBlock %} + +{% /codePreview %} + +{% partial file="/v1.7/connectors/yaml/ingestion-cli.md" /%} + diff --git a/openmetadata-docs/images/connectors/wherescape.webp b/openmetadata-docs/images/connectors/wherescape.webp new file mode 100644 index 00000000000..dc96d27a504 Binary files /dev/null and b/openmetadata-docs/images/connectors/wherescape.webp differ diff --git a/openmetadata-docs/images/v1.7/connectors/wherescape/add-new-service.png b/openmetadata-docs/images/v1.7/connectors/wherescape/add-new-service.png new file mode 100644 index 00000000000..fca57b7f633 Binary files /dev/null and b/openmetadata-docs/images/v1.7/connectors/wherescape/add-new-service.png differ diff --git a/openmetadata-docs/images/v1.7/connectors/wherescape/select-service.png b/openmetadata-docs/images/v1.7/connectors/wherescape/select-service.png new file mode 100644 index 00000000000..ebb585c006b Binary files /dev/null and b/openmetadata-docs/images/v1.7/connectors/wherescape/select-service.png differ diff --git a/openmetadata-docs/images/v1.7/connectors/wherescape/service-connection.png b/openmetadata-docs/images/v1.7/connectors/wherescape/service-connection.png new file mode 100644 index 00000000000..10b13c61e6f Binary files /dev/null and b/openmetadata-docs/images/v1.7/connectors/wherescape/service-connection.png differ