yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-06-27 04:22:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

SSIS Connector (#21290)

Browse Source
This commit is contained in:
Akash Verma 2025-06-02 11:24:49 +05:30 committed by GitHub
parent 066c167d34
commit dba56f306c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
38 changed files with 1705 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
openmetadata-docs/content/partials/v1.7/connectors/pipeline/connectors-list.md
View File

@ -17,5 +17,6 @@
{% connectorInfoCard name="Spline" stage="PROD" href="/connectors/pipeline/spline" platform="OpenMetadata" / %}
{% connectorInfoCard name="Stitch" stage="PROD" href="/connectors/pipeline/stitch" platform="Collate" / %}
{% connectorInfoCard name="Wherescape" stage="BETA" href="/connectors/pipeline/wherescape" platform="Collate" / %}
{% connectorInfoCard name="SSIS" stage="BETA" href="/connectors/pipeline/ssis" platform="Collate" / %}
{% /connectorsListContainer %}

2
openmetadata-docs/content/partials/v1.8/connectors/pipeline/connectors-list.md
View File

@ -17,5 +17,7 @@
{% connectorInfoCard name="Spline" stage="PROD" href="/connectors/pipeline/spline" platform="OpenMetadata" / %}
{% connectorInfoCard name="Stitch" stage="PROD" href="/connectors/pipeline/stitch" platform="Collate" / %}
{% connectorInfoCard name="Wherescape" stage="BETA" href="/connectors/pipeline/wherescape" platform="Collate" / %}
{% connectorInfoCard name="SSIS" stage="BETA" href="/connectors/pipeline/ssis" platform="Collate" / %}
{% /connectorsListContainer %}

4
openmetadata-docs/content/v1.7.x/collate-menu.md
View File

@ -529,6 +529,10 @@ site_menu:
url: /connectors/pipeline/spline/yaml
- category: Connectors / Pipeline / Spline / Troubleshooting
url: /connectors/pipeline/spline/troubleshooting
- category: Connectors / Pipeline / SSIS
url: /connectors/pipeline/SSIS
- category: Connectors / Pipeline / SSIS / Run Externally
url: /connectors/pipeline/SSIS/yaml
- category: Connectors / Pipeline / Stitch
url: /connectors/pipeline/stitch
- category: Connectors / Pipeline / Stitch / Run Externally

132
openmetadata-docs/content/v1.7.x/connectors/pipeline/ssis/index.md Normal file
View File

@ -0,0 +1,132 @@
---
title: SSIS
slug: /connectors/pipeline/ssis
---
{% connectorDetailsHeader
name="SSIS"
stage="BETA"
platform="Collate"
availableFeatures=["Pipelines", "Pipeline Status", "Lineage"]
unavailableFeatures=["Tags", "owners"]
/ %}
In this section, we provide guides and references to use the SSIS connector.
Configure and schedule SSIS metadata workflow from the OpenMetadata UI:
- [Requirements](#requirements)
- [Metadata Ingestion](#metadata-ingestion)
- [Connection Details](#connection-details)
- [Troubleshooting](/connectors/pipeline/ssis/troubleshooting)
{% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/pipeline/ssis/yaml"} /%}
## Requirements
To extract SSIS metadata, we need the batabase connection details where the metadata is stored.
- `API` Permission ( While Creating the User, from Admin -> User )
- To retrieve lineage data, the user must be granted [Component-level permissions](https://docs.matillion.com/metl/docs/2932106/#component).
- To enable lineage tracking in Matillion, **Matillion Enterprise Mode** is required. For detailed setup instructions and further information, refer to the official documentation: [Matillion Lineage Documentation](https://docs.matillion.com/metl/docs/2881895/).
## Metadata Ingestion
{% partial
file="/v1.7/connectors/metadata-ingestion-ui.md"
variables={
connector: "SSIS",
selectServicePath: "/images/v1.7/connectors/ssis/select-service.png",
addNewServicePath: "/images/v1.7/connectors/ssis/add-new-service.png",
serviceConnectionPath: "/images/v1.7/connectors/ssis/service-connection.png",
}
/%}
{% stepsContainer %}
{% extraContent parentTagName="stepsContainer" %}
#### Connection Details
- **Connection**: SSIS metadata database connection.
In terms of `connection` we support the following selections:
- **Microsoft SQL Server**: To connect to the SSIS metadata database:
- Provide the SQL Server connection credentials including username and password
- Specify the database name where SSIS metadata is stored
- Enter the host and port for the SQL Server instance
- The connector will establish a connection to this database to extract SSIS pipeline metadata
To fetch task dependencies and lineage information from your SSIS pipelines, the connector requires access to the SSIS package XML files. You have two options to provide these files:
- **Local Path**: Specify the local directory path where your SSIS package files are stored. The connector will read the XML files directly from this location during metadata extraction.
> **Important:**
> If you are using the **Local Path** option to provide your SSIS package files, you must run the ingestion workflow through the **CLI** instead of the UI. This is because the ingestion process needs direct access to your local filesystem, which is not available when running ingestion jobs from the UI or server.
- **S3 Bucket**: Upload your SSIS project folders containing the package XML files to an S3 bucket. Then, provide the S3 credentials (such as `awsAccessKeyId`, `awsSecretAccessKey`, and region) along with the bucket name in the connector configuration. The connector will retrieve the package files from your S3 storage.
Choose the method that best fits your environment to ensure the connector can access the necessary package XML files for comprehensive metadata and lineage extraction.
- **Note <s3>:**
> When configuring the SSIS connector to extract metadata from S3 storage, you need to upload your SSIS project folders containing all your package files to your S3 bucket.
>
> Typically, SSIS organizes projects in a structure like:
> ```
> repos/
> project1/
> project1/
> ... .dtsx files
> project2/
> project2/
> ... .dtsx files
> ```
>
> You should upload the inner project folders (e.g., `project1/project1/`, `project2/project2/`, etc.) into your S3 bucket. For example, if your bucket name is `packages`, the structure in your bucket should look like:
> ```
> packages/
> project1/
> project1/
> ... .dtsx files
> project2/
> project2/
> ... .dtsx files
> ```
>
> It is necessary to provide the same bucket name (e.g., `packages`) along with the credentials for your S3 storage when configuring the connector.
- **Note <localPath>:**
> When configuring the SSIS connector to extract metadata from a **local path**, you need to provide the path up to the directory containing your SSIS project folders.
>
> For example, if your projects are organized as:
> ```
> /home/user/repos/
> project1/
> project1/
> ... .dtsx files
> project2/
> project2/
> ... .dtsx files
> ```
>
> You should specify the path up to `/home/user/repos/` in your connector configuration. The connector will recursively scan this directory to locate all SSIS project folders and their package XML files.
>
> Ensure that the user running the ingestion workflow has read permissions for the specified local path and all its subdirectories.
- **S3 Storage Service**: To connect to the bucket having xml files:
- Provide the s3 credentials including awsAccessKeyId and awsSecretAccessKey
- Provide the appropriate AWS region (e.g., `us-east-2`) for your S3 bucket.
- Specify the endpoint URL for your S3-compatible storage.
- It is necessary to provide the bucket name containing the SSIS project folders.
{% /extraContent %}
{% partial file="/v1.7/connectors/test-connection.md" /%}
{% partial file="/v1.7/connectors/pipeline/configure-ingestion.md" /%}
{% partial file="/v1.7/connectors/ingestion-schedule-and-deploy.md" /%}
{% /stepsContainer %}

140
openmetadata-docs/content/v1.7.x/connectors/pipeline/ssis/yaml.md Normal file
View File

@ -0,0 +1,140 @@
---
title: Run the SSIS Connector Externally
slug: /connectors/pipeline/ssis/yaml
---
{% connectorDetailsHeader
name="SSIS"
stage="BETA"
platform="Collate"
availableFeatures=["Pipelines", "Pipeline Status", "Lineage"]
unavailableFeatures=["Tags", "Owners"]
/ %}
In this section, we provide guides and references to use the SSIS connector.
Configure and schedule SSIS metadata workflows from the OpenMetadata UI:
- [Requirements](#requirements)
- [Metadata Ingestion](#metadata-ingestion)
{% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%}
## Requirements
### Python Requirements
{% partial file="/v1.7/connectors/python-requirements.md" /%}
To run the SSIS ingestion, you will need to install:
```bash
pip3 install "openmetadata-ingestion[ssis]"
```
{% /note %}
## Metadata Ingestion
All connectors are defined as JSON Schemas.
[Here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/ssisConnection.json)
you can find the structure to create a connection to SSIS.
In order to create and run a Metadata Ingestion workflow, we will follow
the steps to create a YAML configuration able to connect to the source,
process the Entities if needed, and reach the OpenMetadata server.
The workflow is modeled around the following
[JSON Schema](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json)
### 1. Define the YAML Config
This is a sample config for SSIS:
{% codePreview %}
{% codeInfoContainer %}
#### Source Configuration - Service Connection
{% codeInfo srNumber=1 %}
{% /codeInfo %}
{% codeInfo srNumber=1 %}
**connection**:
In terms of `connection` we support the following selections:
- `Mssql`: Pass the required credentials to reach out this service. We will
create a connection to the pointed database and read SSIS data from there.
{% /codeInfo %}
{% partial file="/v1.7/connectors/yaml/pipeline/source-config-def.md" /%}
{% partial file="/v1.7/connectors/yaml/ingestion-sink-def.md" /%}
{% partial file="/v1.7/connectors/yaml/workflow-config-def.md" /%}
{% /codeInfoContainer %}
{% codeBlock fileName="filename.yaml" %}
```yaml {% isCodeBlock=true %}
source:
type: SSIS
serviceName: ssis_source
serviceConnection:
config:
type: SSIS
```
```yaml {% srNumber=6 %}
databaseConnection:
type: Mssql
username: user
password: pass
database: db
hostPort: localhost:1433
```
**storage**:
To extract task dependencies and lineage, the connector needs access to your SSIS package XML files.
You can either provide the local path to your project folders, or upload your projects to an S3 bucket and supply the bucket name along with S3 credentials.
- For s3
Pass the `S3` credentials where your package folders are uploaded and specify the bucket name inside the `bucketNames` field.
```yaml {% srNumber=7 %}
storageConnection:
type: S3
awsConfig:
awsAccessKeyId: test
awsSecretAccessKey: test
awsRegion: us-east-2
endPointURL: https://packages.s3.us-east-2.amazonaws.com
bucketNames:
- bucket_name
```
- For Local
```yaml {% srNumber=8 %}
localProjectsPath: /home/user/repos/
```
{% partial file="/v1.7/connectors/yaml/pipeline/source-config.md" /%}
{% partial file="/v1.7/connectors/yaml/ingestion-sink.md" /%}
{% partial file="/v1.7/connectors/yaml/workflow-config.md" /%}
{% /codeBlock %}
{% /codePreview %}
{% partial file="/v1.7/connectors/yaml/ingestion-cli.md" /%}

4
openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-menu.md
View File

@ -529,6 +529,10 @@ site_menu:
url: /connectors/pipeline/spline/yaml
- category: Connectors / Pipeline / Spline / Troubleshooting
url: /connectors/pipeline/spline/troubleshooting
- category: Connectors / Pipeline / SSIS
url: /connectors/pipeline/SSIS
- category: Connectors / Pipeline / SSIS / Run Externally
url: /connectors/pipeline/SSIS/yaml
- category: Connectors / Pipeline / Stitch
url: /connectors/pipeline/stitch
- category: Connectors / Pipeline / Stitch / Run Externally

132
openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/pipeline/ssis/index.md Normal file
View File

@ -0,0 +1,132 @@
---
title: SSIS
slug: /connectors/pipeline/ssis
---
{% connectorDetailsHeader
name="SSIS"
stage="BETA"
platform="Collate"
availableFeatures=["Pipelines", "Pipeline Status", "Lineage"]
unavailableFeatures=["Tags", "owners"]
/ %}
In this section, we provide guides and references to use the SSIS connector.
Configure and schedule SSIS metadata workflow from the OpenMetadata UI:
- [Requirements](#requirements)
- [Metadata Ingestion](#metadata-ingestion)
- [Connection Details](#connection-details)
- [Troubleshooting](/connectors/pipeline/ssis/troubleshooting)
{% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/pipeline/ssis/yaml"} /%}
## Requirements
To extract SSIS metadata, we need the batabase connection details where the metadata is stored.
- `API` Permission ( While Creating the User, from Admin -> User )
- To retrieve lineage data, the user must be granted [Component-level permissions](https://docs.matillion.com/metl/docs/2932106/#component).
- To enable lineage tracking in Matillion, **Matillion Enterprise Mode** is required. For detailed setup instructions and further information, refer to the official documentation: [Matillion Lineage Documentation](https://docs.matillion.com/metl/docs/2881895/).
## Metadata Ingestion
{% partial
file="/v1.7/connectors/metadata-ingestion-ui.md"
variables={
connector: "SSIS",
selectServicePath: "/images/v1.7/connectors/ssis/select-service.png",
addNewServicePath: "/images/v1.7/connectors/ssis/add-new-service.png",
serviceConnectionPath: "/images/v1.7/connectors/ssis/service-connection.png",
}
/%}
{% stepsContainer %}
{% extraContent parentTagName="stepsContainer" %}
#### Connection Details
- **Connection**: SSIS metadata database connection.
In terms of `connection` we support the following selections:
- **Microsoft SQL Server**: To connect to the SSIS metadata database:
- Provide the SQL Server connection credentials including username and password
- Specify the database name where SSIS metadata is stored
- Enter the host and port for the SQL Server instance
- The connector will establish a connection to this database to extract SSIS pipeline metadata
To fetch task dependencies and lineage information from your SSIS pipelines, the connector requires access to the SSIS package XML files. You have two options to provide these files:
- **Local Path**: Specify the local directory path where your SSIS package files are stored. The connector will read the XML files directly from this location during metadata extraction.
> **Important:**
> If you are using the **Local Path** option to provide your SSIS package files, you must run the ingestion workflow through the **CLI** instead of the UI. This is because the ingestion process needs direct access to your local filesystem, which is not available when running ingestion jobs from the UI or server.
- **S3 Bucket**: Upload your SSIS project folders containing the package XML files to an S3 bucket. Then, provide the S3 credentials (such as `awsAccessKeyId`, `awsSecretAccessKey`, and region) along with the bucket name in the connector configuration. The connector will retrieve the package files from your S3 storage.
Choose the method that best fits your environment to ensure the connector can access the necessary package XML files for comprehensive metadata and lineage extraction.
- **Note <s3>:**
> When configuring the SSIS connector to extract metadata from S3 storage, you need to upload your SSIS project folders containing all your package files to your S3 bucket.
>
> Typically, SSIS organizes projects in a structure like:
> ```
> repos/
> project1/
> project1/
> ... .dtsx files
> project2/
> project2/
> ... .dtsx files
> ```
>
> You should upload the inner project folders (e.g., `project1/project1/`, `project2/project2/`, etc.) into your S3 bucket. For example, if your bucket name is `packages`, the structure in your bucket should look like:
> ```
> packages/
> project1/
> project1/
> ... .dtsx files
> project2/
> project2/
> ... .dtsx files
> ```
>
> It is necessary to provide the same bucket name (e.g., `packages`) along with the credentials for your S3 storage when configuring the connector.
- **Note <localPath>:**
> When configuring the SSIS connector to extract metadata from a **local path**, you need to provide the path up to the directory containing your SSIS project folders.
>
> For example, if your projects are organized as:
> ```
> /home/user/repos/
> project1/
> project1/
> ... .dtsx files
> project2/
> project2/
> ... .dtsx files
> ```
>
> You should specify the path up to `/home/user/repos/` in your connector configuration. The connector will recursively scan this directory to locate all SSIS project folders and their package XML files.
>
> Ensure that the user running the ingestion workflow has read permissions for the specified local path and all its subdirectories.
- **S3 Storage Service**: To connect to the bucket having xml files:
- Provide the s3 credentials including awsAccessKeyId and awsSecretAccessKey
- Provide the appropriate AWS region (e.g., `us-east-2`) for your S3 bucket.
- Specify the endpoint URL for your S3-compatible storage.
- It is necessary to provide the bucket name containing the SSIS project folders.
{% /extraContent %}
{% partial file="/v1.7/connectors/test-connection.md" /%}
{% partial file="/v1.7/connectors/pipeline/configure-ingestion.md" /%}
{% partial file="/v1.7/connectors/ingestion-schedule-and-deploy.md" /%}
{% /stepsContainer %}

140
openmetadata-docs/content/v1.8.x-SNAPSHOT/connectors/pipeline/ssis/yaml.md Normal file
View File

@ -0,0 +1,140 @@
---
title: Run the SSIS Connector Externally
slug: /connectors/pipeline/ssis/yaml
---
{% connectorDetailsHeader
name="SSIS"
stage="BETA"
platform="Collate"
availableFeatures=["Pipelines", "Pipeline Status", "Lineage"]
unavailableFeatures=["Tags", "Owners"]
/ %}
In this section, we provide guides and references to use the SSIS connector.
Configure and schedule SSIS metadata workflows from the OpenMetadata UI:
- [Requirements](#requirements)
- [Metadata Ingestion](#metadata-ingestion)
{% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%}
## Requirements
### Python Requirements
{% partial file="/v1.7/connectors/python-requirements.md" /%}
To run the SSIS ingestion, you will need to install:
```bash
pip3 install "openmetadata-ingestion[ssis]"
```
{% /note %}
## Metadata Ingestion
All connectors are defined as JSON Schemas.
[Here](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/ssisConnection.json)
you can find the structure to create a connection to SSIS.
In order to create and run a Metadata Ingestion workflow, we will follow
the steps to create a YAML configuration able to connect to the source,
process the Entities if needed, and reach the OpenMetadata server.
The workflow is modeled around the following
[JSON Schema](https://github.com/open-metadata/OpenMetadata/blob/main/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json)
### 1. Define the YAML Config
This is a sample config for SSIS:
{% codePreview %}
{% codeInfoContainer %}
#### Source Configuration - Service Connection
{% codeInfo srNumber=1 %}
{% /codeInfo %}
{% codeInfo srNumber=1 %}
**connection**:
In terms of `connection` we support the following selections:
- `Mssql`: Pass the required credentials to reach out this service. We will
create a connection to the pointed database and read SSIS data from there.
{% /codeInfo %}
{% partial file="/v1.7/connectors/yaml/pipeline/source-config-def.md" /%}
{% partial file="/v1.7/connectors/yaml/ingestion-sink-def.md" /%}
{% partial file="/v1.7/connectors/yaml/workflow-config-def.md" /%}
{% /codeInfoContainer %}
{% codeBlock fileName="filename.yaml" %}
```yaml {% isCodeBlock=true %}
source:
type: SSIS
serviceName: ssis_source
serviceConnection:
config:
type: SSIS
```
```yaml {% srNumber=6 %}
databaseConnection:
type: Mssql
username: user
password: pass
database: db
hostPort: localhost:1433
```
**storage**:
To extract task dependencies and lineage, the connector needs access to your SSIS package XML files.
You can either provide the local path to your project folders, or upload your projects to an S3 bucket and supply the bucket name along with S3 credentials.
- For s3
Pass the `S3` credentials where your package folders are uploaded and specify the bucket name inside the `bucketNames` field.
```yaml {% srNumber=7 %}
storageConnection:
type: S3
awsConfig:
awsAccessKeyId: test
awsSecretAccessKey: test
awsRegion: us-east-2
endPointURL: https://packages.s3.us-east-2.amazonaws.com
bucketNames:
- bucket_name
```
- For Local
```yaml {% srNumber=8 %}
localProjectsPath: /home/user/repos/
```
{% partial file="/v1.7/connectors/yaml/pipeline/source-config.md" /%}
{% partial file="/v1.7/connectors/yaml/ingestion-sink.md" /%}
{% partial file="/v1.7/connectors/yaml/workflow-config.md" /%}
{% /codeBlock %}
{% /codePreview %}
{% partial file="/v1.7/connectors/yaml/ingestion-cli.md" /%}

BIN
openmetadata-docs/images/connectors/ssis.webp Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 9.3 KiB

BIN
openmetadata-docs/images/v1.7/connectors/ssis/add-new-service.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 98 KiB

BIN
openmetadata-docs/images/v1.7/connectors/ssis/select-service.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 245 KiB

BIN
openmetadata-docs/images/v1.7/connectors/ssis/service-connection.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 78 KiB

BIN
openmetadata-docs/images/v1.8/connectors/ssis/add-new-service.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 98 KiB

BIN
openmetadata-docs/images/v1.8/connectors/ssis/select-service.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 245 KiB

BIN
openmetadata-docs/images/v1.8/connectors/ssis/service-connection.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 78 KiB

2
openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java
View File

@ -46,6 +46,7 @@ import org.openmetadata.schema.services.connections.mlmodel.VertexAIConnection;
import org.openmetadata.schema.services.connections.pipeline.AirflowConnection;
import org.openmetadata.schema.services.connections.pipeline.MatillionConnection;
import org.openmetadata.schema.services.connections.pipeline.NifiConnection;
import org.openmetadata.schema.services.connections.pipeline.SSISConnection;
import org.openmetadata.schema.services.connections.search.ElasticSearchConnection;
import org.openmetadata.schema.services.connections.storage.GCSConnection;
@ -96,6 +97,7 @@ public final class ClassConverterFactory {
Map.entry(VertexAIConnection.class, new VertexAIConnectionClassConverter()));
Map.entry(Workflow.class, new WorkflowClassConverter());
Map.entry(CassandraConnection.class, new CassandraConnectionClassConverter());
Map.entry(SSISConnection.class, new SsisConnectionClassConverter());
}
public static ClassConverter getConverter(Class<?> clazz) {

27
openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/SsisConnectionClassConverter.java Normal file
View File

@ -0,0 +1,27 @@
package org.openmetadata.service.secrets.converter;
import java.util.List;
import org.openmetadata.schema.services.connections.pipeline.SSISConnection;
import org.openmetadata.schema.services.connections.storage.S3Connection;
import org.openmetadata.service.util.JsonUtils;
/** Converter class to get an `SsisConnection` object. */
public class SsisConnectionClassConverter extends ClassConverter {
private static final List<Class<?>> CONFIG_SOURCE_CLASSES =
List.of(S3Connection.class, String.class);
public SsisConnectionClassConverter() {
super(SSISConnection.class);
}
@Override
public Object convert(Object object) {
SSISConnection ssisConnection = (SSISConnection) JsonUtils.convertValue(object, this.clazz);
tryToConvertOrFail(ssisConnection.getPackageConnection(), CONFIG_SOURCE_CLASSES)
.ifPresent(ssisConnection::setPackageConnection);
return ssisConnection;
}
}

44
openmetadata-service/src/main/resources/json/data/testConnections/pipeline/ssis.json Normal file
View File

@ -0,0 +1,44 @@
{
"name": "SSIS",
"displayName": "SSIS Test Connection",
"description": "This Test Connection validates the access against the server and basic metadata extraction of pipelines.",
"steps": [
{
"name": "CheckAccess",
"description": "Check if the database is reachable to fetch the DAG details.",
"errorMessage": "Failed to connect to ssis, please validate the credentials",
"shortCircuit": true,
"mandatory": true
},
{
"name": "SSISDBExist",
"description": "Check if the SSISDB database exists and is accessible.",
"errorMessage": "SSISDB database does not exist or is not accessible.",
"shortCircuit": true,
"mandatory": true
},
{
"name": "SSISDBProjectsExist",
"description": "Check if there are any projects present in the SSISDB database.",
"errorMessage": "No projects found in SSISDB database.",
"shortCircuit": true,
"mandatory": true
},
{
"name": "SSISDBPackagesExist",
"description": "Check if there are any packages present in the SSISDB database.",
"errorMessage": "No packages found in SSISDB database.",
"shortCircuit": true,
"mandatory": true
},
{
"name": "CheckS3Access",
"description": "Check storage access",
"errorMessage": "Unable to connect to storage service",
"shortCircuit": true,
"mandatory": false
}
]
}

53
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/ssisConnection.json Normal file
View File

@ -0,0 +1,53 @@
{
"$id": "https://open-metadata.org/schema/entity/services/connections/pipeline/ssisConnection.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "SSISConnection",
"description": "SSIS Metadata Database Connection Config",
"type": "object",
"javaType": "org.openmetadata.schema.services.connections.pipeline.SSISConnection",
"definitions": {
"SSISType": {
"description": "Service type.",
"type": "string",
"enum": ["SSIS"],
"default": "SSIS"
},
"localProjectsPath": {
"title": "Local Projects Path",
"description": "Path leading to your projects",
"type": "string"
}
},
"properties": {
"type": {
"title": "Service Type",
"description": "Service Type",
"$ref": "#/definitions/SSISType",
"default": "SSIS"
},
"databaseConnection": {
"title": "Metadata Database Connection",
"description": "Underlying database connection",
"$ref": "../database/mssqlConnection.json"
},
"packageConnection": {
"title": "SSIS Projects Location (Local Path or S3 Bucket)",
"description": "Underlying storage connection",
"oneOf": [
{
"$ref": "../storage/s3Connection.json"
},
{
"$ref": "#/definitions/localProjectsPath"
}
]
},
"supportsMetadataExtraction": {
"title": "Supports Metadata Extraction",
"$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
}
},
"additionalProperties": false,
"required": ["databaseConnection", "packageConnection"]
}

9
openmetadata-spec/src/main/resources/json/schema/entity/services/pipelineService.json
View File

@ -35,7 +35,8 @@
"Matillion",
"Stitch",
"DataFactory",
"Wherescape"
"Wherescape",
"SSIS"
],
"javaEnums": [
{
@ -94,6 +95,9 @@
},
{
"name": "Wherescape"
},
{
"name": "SSIS"
}
]
},
@ -114,6 +118,9 @@
{
"$ref": "./connections/pipeline/wherescapeConnection.json"
},
{
"$ref": "./connections/pipeline/ssisConnection.json"
},
{
"$ref": "./connections/pipeline/gluePipelineConnection.json"
},

252
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/SSIS.md Normal file
View File

@ -0,0 +1,252 @@
# SSIS
In this section, we provide guides and references to use the SSIS connector.
## Requirements
1. **SSIS Connector**: which we will configure in this section and requires access to the underlying database.
## Connection Details
$$section
### Metadata Database Connection $(id="connection")
SSIS uses MSSQL as its database.
$$
## Mssql Connection
In this section, we provide guides and references to use the MSSQL connection.
## Requirements
The user must have `SELECT` privileges to fetch the metadata of tables and views.
```sql
-- Create a new user
-- More details https://learn.microsoft.com/en-us/sql/t-sql/statements/create-user-transact-sql?view=sql-server-ver16
CREATE USER Mary WITH PASSWORD = '********';
-- Grant SELECT on table
GRANT SELECT TO Mary;
```
### Remote Connection
#### 1. SQL Server running
Make sure the SQL server that you are trying to connect is in running state.
#### 2. Allow remote connection on MSSMS (Microsoft SQL Server Management Studio)
This step allow the sql server to accept remote connection request.
![remote-connection](/doc-images/Database/Mssql/remote-connection.png)
#### 3. Configure Windows Firewall
If you are using SQL server on Windows, you must configure the firewall on the computer running SQL Server to allow access.
1. On the Start menu, select `Run`, type `WF.msc`, and then select `OK`.
2. In the `Windows Firewall with Advanced Security`, in the left pane, right-click` Inbound Rules`, and then select `New Rule` in the action pane.
3. In the `Rule Type` dialog box, select `Port`, and then select `Next`.
4. In the `Protocol and Ports` dialog box, select `TCP`. Select Specific local ports, and then type the port number of the instance of the Database Engine, such as 1433 for the default instance. Select `Next`.
5. In the `Action` dialog box, select `Allow` the connection, and then select Next.
6. In the `Profile` dialog box, select any profiles that describe the computer connection environment when you want to connect to the Database Engine, and then select `Next`.
7. In the `Name` dialog box, type a name and description for this rule, and then select `Finish`.
For details step please refer this [link](https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/configure-a-windows-firewall-for-database-engine-access?view=sql-server-ver15).
You can find further information on the MSSQL connector in the [docs](https://docs.open-metadata.org/connectors/database/mssql).
## Connection Details
$$section
### Scheme $(id="scheme")
There are three schemes based on the user's requirement to fetch data from MSSQL:
- **mssql+pytds**: High-performance open-source library for connecting to Microsoft SQL Server.
- **mssql+pyodbc**: Cross-platform Python library that uses ODBC to connect to Microsoft SQL Server.
- **mssql+pymssql**: Python library that uses FreeTDS to connect to Microsoft SQL Server, with support for bulk data transfer and query timeouts.
If you are connecting via windows authentication from a linux docker deployment please use `mssql+pymssql`.
$$
$$section
### Username $(id="username")
Username to connect to MSSQL. This user should have privileges to read all the metadata in MSSQL.
$$
$$section
### Password $(id="password")
Password to connect to MSSQL.
$$
$$section
### Host Port $(id="hostPort")
This parameter specifies the host and port of the MSSQL instance. This should be specified as a string in the format `hostname:port`. For example, you might set the hostPort parameter to `localhost:1433`.
If you are running the OpenMetadata ingestion in a docker and your services are hosted on the `localhost`, then use `host.docker.internal:1433` as the value.
$$
$$section
### Database $(id="database")
Initial Mssql database to connect to.
$$
$$section
### Driver $(id="driver")
Connecting to MSSQL via **pyodbc** scheme requires the ODBC driver to be installed. Specify ODBC driver name in the field.
You can download the ODBC driver from [here](https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver16).
In case of Docker or Kubernetes deployments, this driver comes out of the box with version `ODBC Driver 18 for SQL Server`.
$$
$$section
### Project Location $(id="packageConnection")
You have two options to provide your SSIS projects for ingestion:
1. **Local Path**:<span style="font-size: 1.1em; color: #b22222; font-style: italic; font-weight: bold;">If you are using the Local Path option, you must run the ingestion workflow through the CLI instead of the UI.</span>
2. **S3 Bucket**: Upload your SSIS projects to an S3 bucket and provide the bucket name along with the necessary S3 credentials.
You can choose either of these methods based on your setup and requirements.
$$
## S3Connection
In this section, we provide guides and references to use the S3 connector.
## Requirements
We need the following permissions in AWS:
### S3 Permissions
For all the buckets that we want to ingest, we need to provide the following:
- `s3:ListBucket`
- `s3:GetObject`
Note that the `Resources` should be all the buckets that you'd like to scan. A possible policy could be:
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:ListBucket",
],
"Resource": [
"arn:aws:s3:::*"
]
}
]
}
```
## Connection Details
$$section
### AWS Access Key ID $(id="awsAccessKeyId")
When you interact with AWS, you specify your AWS security credentials to verify who you are and whether you have permission to access the resources that you are requesting. AWS uses the security credentials to authenticate and authorize your requests ([docs](https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html)).
Access keys consist of two parts:
1. An access key ID (for example, `AKIAIOSFODNN7EXAMPLE`),
2. And a secret access key (for example, `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY`).
You must use both the access key ID and secret access key together to authenticate your requests.
You can find further information on how to manage your access keys [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html)
$$
$$section
### AWS Secret Access Key $(id="awsSecretAccessKey")
Secret access key (for example, `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY`).
$$
$$section
### AWS Region $(id="awsRegion")
Each AWS Region is a separate geographic area in which AWS clusters data centers ([docs](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html)).
As AWS can have instances in multiple regions, we need to know the region the service you want reach belongs to.
Note that the AWS Region is the only required parameter when configuring a connection. When connecting to the services programmatically, there are different ways in which we can extract and use the rest of AWS configurations. You can find further information about configuring your credentials [here](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials).
$$
$$section
### AWS Session Token $(id="awsSessionToken")
If you are using temporary credentials to access your services, you will need to inform the AWS Access Key ID and AWS Secrets Access Key. Also, these will include an AWS Session Token.
You can find more information on [Using temporary credentials with AWS resources](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html).
$$
$$section
### Endpoint URL $(id="endPointURL")
To connect programmatically to an AWS service, you use an endpoint. An *endpoint* is the URL of the entry point for an AWS web service. The AWS SDKs and the AWS Command Line Interface (AWS CLI) automatically use the default endpoint for each service in an AWS Region. But you can specify an alternate endpoint for your API requests.
Find more information on [AWS service endpoints](https://docs.aws.amazon.com/general/latest/gr/rande.html).
$$
$$section
### Profile Name $(id="profileName")
A named profile is a collection of settings and credentials that you can apply to an AWS CLI command. When you specify a profile to run a command, the settings and credentials are used to run that command. Multiple named profiles can be stored in the config and credentials files.
You can inform this field if you'd like to use a profile other than `default`.
Find here more information about [Named profiles for the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html).
$$
$$section
### Assume Role ARN $(id="assumeRoleArn")
Typically, you use `AssumeRole` within your account or for cross-account access. In this field you'll set the `ARN` (Amazon Resource Name) of the policy of the other account.
A user who wants to access a role in a different account must also have permissions that are delegated from the account administrator. The administrator must attach a policy that allows the user to call `AssumeRole` for the `ARN` of the role in the other account.
This is a required field if you'd like to `AssumeRole`.
Find more information on [AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html).
$$
$$section
### Assume Role Session Name $(id="assumeRoleSessionName")
An identifier for the assumed role session. Use the role session name to uniquely identify a session when the same role is assumed by different principals or for different reasons.
By default, we'll use the name `OpenMetadataSession`.
Find more information about the [Role Session Name](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#:~:text=An%20identifier%20for%20the%20assumed%20role%20session.).
$$
$$section
### Assume Role Source Identity $(id="assumeRoleSourceIdentity")
The source identity specified by the principal that is calling the `AssumeRole` operation. You can use source identity information in AWS CloudTrail logs to determine who took actions with a role.
Find more information about [Source Identity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#:~:text=Required%3A%20No-,SourceIdentity,-The%20source%20identity).
$$
$$section
### Bucket Name $(id="bucketNames")
Provide the name of the bucket that contains your project folders
$$

1
openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts
View File

@ -428,6 +428,7 @@ export const BETA_SERVICES = [
MetadataServiceType.AlationSink,
DatabaseServiceType.Cockroach,
SearchServiceType.OpenSearch,
PipelineServiceType.Ssis,
];
export const TEST_CONNECTION_INITIAL_MESSAGE = i18n.t(

54
openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts
View File

@ -319,6 +319,8 @@ export interface RequestConnection {
*
* Wherescape Metadata Database Connection Config
*
* SSIS Metadata Database Connection Config
*
* Glue Pipeline Connection Config
*
* Airbyte Metadata Database Connection Config
@ -964,6 +966,10 @@ export interface ConfigClass {
* SSL Configuration for OpenMetadata Server
*/
sslConfig?: SSLConfigObject;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
/**
* How to run the SQLite database. :memory: by default.
*/
@ -1437,6 +1443,10 @@ export interface ConfigClass {
* Underlying database connection
*/
databaseConnection?: DatabaseConnectionClass;
/**
* Underlying storage connection
*/
packageConnection?: S3Connection | string;
/**
* Fivetran API Secret.
*/
@ -1782,10 +1792,10 @@ export interface UsernamePasswordAuthentication {
*
* Regex exclude pipelines.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch containers that matches the pattern.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch search indexes that matches the pattern.
*/
export interface FilterPattern {
@ -2816,6 +2826,10 @@ export interface ConfigConnection {
* as the database name.
*/
databaseName?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
/**
* Regex exclude pipelines.
*/
@ -3311,6 +3325,10 @@ export interface HiveMetastoreConnectionDetails {
* attempts to scan all the schemas.
*/
databaseSchema?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
}
/**
@ -3389,6 +3407,37 @@ export interface OracleConnectionType {
[property: string]: any;
}
/**
* S3 Connection.
*/
export interface S3Connection {
awsConfig: AWSCredentials;
/**
* Bucket Names of the data source.
*/
bucketNames?: string[];
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Regex to only fetch containers that matches the pattern.
*/
containerFilterPattern?: FilterPattern;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: S3Type;
}
/**
* Service Type
*
* S3 service type
*/
export enum S3Type {
S3 = "S3",
}
/**
* Source to get the .pbit files to extract lineage information
*
@ -3870,6 +3919,7 @@ export enum RESTType {
Snowflake = "Snowflake",
Spark = "Spark",
Spline = "Spline",
Ssis = "SSIS",
Stitch = "Stitch",
Superset = "Superset",
Synapse = "Synapse",

42
openmetadata-ui/src/main/resources/ui/src/generated/api/services/createPipelineService.ts
View File

@ -70,6 +70,8 @@ export interface PipelineConnection {
*
* Wherescape Metadata Database Connection Config
*
* SSIS Metadata Database Connection Config
*
* Glue Pipeline Connection Config
*
* Airbyte Metadata Database Connection Config
@ -147,7 +149,11 @@ export interface ConfigClass {
* Underlying database connection
*/
databaseConnection?: DatabaseConnectionClass;
awsConfig?: AWSCredentials;
/**
* Underlying storage connection
*/
packageConnection?: S3Connection | string;
awsConfig?: AWSCredentials;
/**
* Airbyte API version.
*/
@ -572,6 +578,8 @@ export interface AuthConfigurationType {
* Regex to only include/exclude schemas that matches the pattern.
*
* Regex to only include/exclude tables that matches the pattern.
*
* Regex to only fetch containers that matches the pattern.
*/
export interface FilterPattern {
/**
@ -845,6 +853,37 @@ export interface NifiCredentialsConfiguration {
clientkeyPath?: string;
}
/**
* S3 Connection.
*/
export interface S3Connection {
awsConfig: AWSCredentials;
/**
* Bucket Names of the data source.
*/
bucketNames?: string[];
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Regex to only fetch containers that matches the pattern.
*/
containerFilterPattern?: FilterPattern;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: S3Type;
}
/**
* Service Type
*
* S3 service type
*/
export enum S3Type {
S3 = "S3",
}
/**
* SASL Configuration details.
*
@ -914,6 +953,7 @@ export enum PipelineServiceType {
OpenLineage = "OpenLineage",
Spark = "Spark",
Spline = "Spline",
Ssis = "SSIS",
Stitch = "Stitch",
Wherescape = "Wherescape",
}

38
openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts
View File

@ -2035,6 +2035,8 @@ export interface ServiceConnection {
*
* Wherescape Metadata Database Connection Config
*
* SSIS Metadata Database Connection Config
*
* Glue Pipeline Connection Config
*
* Airbyte Metadata Database Connection Config
@ -3322,6 +3324,10 @@ export interface ConfigClass {
* Underlying database connection
*/
databaseConnection?: DatabaseConnectionClass;
/**
* Underlying storage connection
*/
packageConnection?: S3Connection | string;
/**
* Fivetran API Secret.
*/
@ -4874,6 +4880,37 @@ export interface OracleConnectionType {
[property: string]: any;
}
/**
* S3 Connection.
*/
export interface S3Connection {
awsConfig: AWSCredentials;
/**
* Bucket Names of the data source.
*/
bucketNames?: string[];
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Regex to only fetch containers that matches the pattern.
*/
containerFilterPattern?: FilterPattern;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: S3Type;
}
/**
* Service Type
*
* S3 service type
*/
export enum S3Type {
S3 = "S3",
}
/**
* Source to get the .pbit files to extract lineage information
*
@ -5342,6 +5379,7 @@ export enum RESTType {
Snowflake = "Snowflake",
Spark = "Spark",
Spline = "Spline",
Ssis = "SSIS",
Stitch = "Stitch",
Superset = "Superset",
Synapse = "Synapse",

42
openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts
View File

@ -201,6 +201,8 @@ export interface TestServiceConnectionConnection {
*
* Wherescape Metadata Database Connection Config
*
* SSIS Metadata Database Connection Config
*
* Glue Pipeline Connection Config
*
* Airbyte Metadata Database Connection Config
@ -1323,6 +1325,10 @@ export interface ConfigClass {
* Underlying database connection
*/
databaseConnection?: DatabaseConnectionClass;
/**
* Underlying storage connection
*/
packageConnection?: S3Connection | string;
/**
* Fivetran API Secret.
*/
@ -1668,10 +1674,10 @@ export interface UsernamePasswordAuthentication {
*
* Regex exclude pipelines.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch containers that matches the pattern.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch search indexes that matches the pattern.
*/
export interface FilterPattern {
@ -3283,6 +3289,37 @@ export interface OracleConnectionType {
[property: string]: any;
}
/**
* S3 Connection.
*/
export interface S3Connection {
awsConfig: AWSCredentials;
/**
* Bucket Names of the data source.
*/
bucketNames?: string[];
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Regex to only fetch containers that matches the pattern.
*/
containerFilterPattern?: FilterPattern;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: S3Type;
}
/**
* Service Type
*
* S3 service type
*/
export enum S3Type {
S3 = "S3",
}
/**
* Source to get the .pbit files to extract lineage information
*
@ -3764,6 +3801,7 @@ export enum RESTType {
Snowflake = "Snowflake",
Spark = "Spark",
Spline = "Spline",
Ssis = "SSIS",
Stitch = "Stitch",
Superset = "Superset",
Synapse = "Synapse",

54
openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts
View File

@ -404,10 +404,10 @@ export enum AuthProvider {
*
* Regex exclude pipelines.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch containers that matches the pattern.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch search indexes that matches the pattern.
*/
export interface FilterPattern {
@ -735,6 +735,8 @@ export interface RequestConnection {
*
* Wherescape Metadata Database Connection Config
*
* SSIS Metadata Database Connection Config
*
* Glue Pipeline Connection Config
*
* Airbyte Metadata Database Connection Config
@ -1380,6 +1382,10 @@ export interface ConfigClass {
* SSL Configuration for OpenMetadata Server
*/
sslConfig?: SSLConfigObject;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
/**
* How to run the SQLite database. :memory: by default.
*/
@ -1853,6 +1859,10 @@ export interface ConfigClass {
* Underlying database connection
*/
databaseConnection?: DatabaseConnectionClass;
/**
* Underlying storage connection
*/
packageConnection?: S3Connection | string;
/**
* Fivetran API Secret.
*/
@ -3143,6 +3153,10 @@ export interface ConfigConnection {
* as the database name.
*/
databaseName?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
/**
* Regex exclude pipelines.
*/
@ -3627,6 +3641,10 @@ export interface HiveMetastoreConnectionDetails {
* attempts to scan all the schemas.
*/
databaseSchema?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
}
/**
@ -3705,6 +3723,37 @@ export interface OracleConnectionType {
[property: string]: any;
}
/**
* S3 Connection.
*/
export interface S3Connection {
awsConfig: AWSCredentials;
/**
* Bucket Names of the data source.
*/
bucketNames?: string[];
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Regex to only fetch containers that matches the pattern.
*/
containerFilterPattern?: FilterPattern;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: S3Type;
}
/**
* Service Type
*
* S3 service type
*/
export enum S3Type {
S3 = "S3",
}
/**
* Source to get the .pbit files to extract lineage information
*
@ -4142,6 +4191,7 @@ export enum RESTType {
Snowflake = "Snowflake",
Spark = "Spark",
Spline = "Spline",
Ssis = "SSIS",
Stitch = "Stitch",
Superset = "Superset",
Synapse = "Synapse",

1
openmetadata-ui/src/main/resources/ui/src/generated/entity/data/pipeline.ts
View File

@ -506,6 +506,7 @@ export enum PipelineServiceType {
OpenLineage = "OpenLineage",
Spark = "Spark",
Spline = "Spline",
Ssis = "SSIS",
Stitch = "Stitch",
Wherescape = "Wherescape",
}

4
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/dashboard/supersetConnection.ts
View File

@ -176,6 +176,10 @@ export interface SupersetConnectionClass {
* attempts to scan all the schemas.
*/
databaseSchema?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
}
/**

4
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/hiveConnection.ts
View File

@ -206,6 +206,10 @@ export interface HiveMetastoreConnectionDetails {
* attempts to scan all the schemas.
*/
databaseSchema?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
}
/**

4
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/metadata/alationConnection.ts
View File

@ -197,6 +197,10 @@ export interface AlationDatabaseConnection {
* attempts to scan all the schemas.
*/
databaseSchema?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
}
/**

4
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/pipeline/airflowConnection.ts
View File

@ -124,6 +124,10 @@ export interface MetadataDatabaseConnection {
* Username to connect to SQLite. Blank for in-memory database.
*/
username?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
/**
* Custom OpenMetadata Classification name for Postgres policy tags.
*/

300
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/pipeline/ssisConnection.ts Normal file
View File

@ -0,0 +1,300 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* SSIS Metadata Database Connection Config
*/
export interface SsisConnection {
/**
* Underlying database connection
*/
databaseConnection: MssqlConnection;
/**
* Underlying storage connection
*/
packageConnection: S3Connection | string;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: SSISType;
}
/**
* Underlying database connection
*
* Mssql Database Connection Config
*/
export interface MssqlConnection {
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Database of the data source. This is optional parameter, if you would like to restrict
* the metadata reading to a single database. When left blank, OpenMetadata Ingestion
* attempts to scan all the databases.
*/
database: string;
/**
* Regex to only include/exclude databases that matches the pattern.
*/
databaseFilterPattern?: FilterPattern;
/**
* ODBC driver version in case of pyodbc connection.
*/
driver?: string;
/**
* Host and port of the MSSQL service.
*/
hostPort?: string;
/**
* Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this.
*/
ingestAllDatabases?: boolean;
/**
* Password to connect to MSSQL.
*/
password?: string;
sampleDataStorageConfig?: SampleDataStorageConfig;
/**
* Regex to only include/exclude schemas that matches the pattern.
*/
schemaFilterPattern?: FilterPattern;
/**
* SQLAlchemy driver scheme options.
*/
scheme?: MssqlScheme;
supportsDatabase?: boolean;
supportsDataDiff?: boolean;
supportsDBTExtraction?: boolean;
supportsLineageExtraction?: boolean;
supportsMetadataExtraction?: boolean;
supportsProfiler?: boolean;
supportsQueryComment?: boolean;
supportsUsageExtraction?: boolean;
/**
* Regex to only include/exclude tables that matches the pattern.
*/
tableFilterPattern?: FilterPattern;
/**
* Service Type
*/
type?: MssqlType;
/**
* Username to connect to MSSQL. This user should have privileges to read all the metadata
* in MsSQL.
*/
username?: string;
}
/**
* Regex to only include/exclude databases that matches the pattern.
*
* Regex to only fetch entities that matches the pattern.
*
* Regex to only include/exclude schemas that matches the pattern.
*
* Regex to only include/exclude tables that matches the pattern.
*
* Regex to only fetch containers that matches the pattern.
*/
export interface FilterPattern {
/**
* List of strings/regex patterns to match and exclude only database entities that match.
*/
excludes?: string[];
/**
* List of strings/regex patterns to match and include only database entities that match.
*/
includes?: string[];
}
/**
* Storage config to store sample data
*/
export interface SampleDataStorageConfig {
config?: DataStorageConfig;
}
/**
* Storage config to store sample data
*/
export interface DataStorageConfig {
/**
* Bucket Name
*/
bucketName?: string;
/**
* Provide the pattern of the path where the generated sample data file needs to be stored.
*/
filePathPattern?: string;
/**
* When this field enabled a single parquet file will be created to store sample data,
* otherwise we will create a new file per day
*/
overwriteData?: boolean;
/**
* Prefix of the data source.
*/
prefix?: string;
storageConfig?: AwsCredentials;
[property: string]: any;
}
/**
* AWS credentials configs.
*/
export interface AwsCredentials {
/**
* The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume
* Role
*/
assumeRoleArn?: string;
/**
* An identifier for the assumed role session. Use the role session name to uniquely
* identify a session when the same role is assumed by different principals or for different
* reasons. Required Field in case of Assume Role
*/
assumeRoleSessionName?: string;
/**
* The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume
* Role
*/
assumeRoleSourceIdentity?: string;
/**
* AWS Access key ID.
*/
awsAccessKeyId?: string;
/**
* AWS Region
*/
awsRegion?: string;
/**
* AWS Secret Access Key.
*/
awsSecretAccessKey?: string;
/**
* AWS Session Token.
*/
awsSessionToken?: string;
/**
* EndPoint URL for the AWS
*/
endPointURL?: string;
/**
* The name of a profile to use with the boto session.
*/
profileName?: string;
}
/**
* SQLAlchemy driver scheme options.
*/
export enum MssqlScheme {
MssqlPymssql = "mssql+pymssql",
MssqlPyodbc = "mssql+pyodbc",
MssqlPytds = "mssql+pytds",
}
/**
* Service Type
*
* Service type.
*/
export enum MssqlType {
Mssql = "Mssql",
}
/**
* S3 Connection.
*/
export interface S3Connection {
awsConfig: AWSCredentials;
/**
* Bucket Names of the data source.
*/
bucketNames?: string[];
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Regex to only fetch containers that matches the pattern.
*/
containerFilterPattern?: FilterPattern;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: S3Type;
}
/**
* AWS credentials configs.
*/
export interface AWSCredentials {
/**
* The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume
* Role
*/
assumeRoleArn?: string;
/**
* An identifier for the assumed role session. Use the role session name to uniquely
* identify a session when the same role is assumed by different principals or for different
* reasons. Required Field in case of Assume Role
*/
assumeRoleSessionName?: string;
/**
* The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume
* Role
*/
assumeRoleSourceIdentity?: string;
/**
* AWS Access key ID.
*/
awsAccessKeyId?: string;
/**
* AWS Region
*/
awsRegion: string;
/**
* AWS Secret Access Key.
*/
awsSecretAccessKey?: string;
/**
* AWS Session Token.
*/
awsSessionToken?: string;
/**
* EndPoint URL for the AWS
*/
endPointURL?: string;
/**
* The name of a profile to use with the boto session.
*/
profileName?: string;
}
/**
* Service Type
*
* S3 service type
*/
export enum S3Type {
S3 = "S3",
}
/**
* Service Type
*
* Service type.
*/
export enum SSISType {
Ssis = "SSIS",
}

54
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts
View File

@ -196,6 +196,8 @@ export interface ServiceConnectionClass {
*
* Wherescape Metadata Database Connection Config
*
* SSIS Metadata Database Connection Config
*
* Glue Pipeline Connection Config
*
* Airbyte Metadata Database Connection Config
@ -1046,6 +1048,10 @@ export interface ConfigClass {
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
/**
* How to run the SQLite database. :memory: by default.
*/
@ -1479,6 +1485,10 @@ export interface ConfigClass {
* Underlying database connection
*/
databaseConnection?: DatabaseConnectionClass;
/**
* Underlying storage connection
*/
packageConnection?: S3Connection | string;
/**
* Fivetran API Secret.
*/
@ -1647,10 +1657,10 @@ export interface UsernamePasswordAuthentication {
*
* Regex exclude pipelines.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch containers that matches the pattern.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch search indexes that matches the pattern.
*/
export interface FilterPattern {
@ -2676,6 +2686,10 @@ export interface ConfigConnection {
* attempts to scan all the schemas.
*/
databaseSchema?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
/**
* HDB Store User Key generated from the command `hdbuserstore SET <KEY> <host:port>
* <USERNAME> <PASSWORD>`
@ -3176,6 +3190,10 @@ export interface HiveMetastoreConnectionDetails {
* attempts to scan all the schemas.
*/
databaseSchema?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
}
/**
@ -3254,6 +3272,37 @@ export interface OracleConnectionType {
[property: string]: any;
}
/**
* S3 Connection.
*/
export interface S3Connection {
awsConfig: AWSCredentials;
/**
* Bucket Names of the data source.
*/
bucketNames?: string[];
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Regex to only fetch containers that matches the pattern.
*/
containerFilterPattern?: FilterPattern;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: S3Type;
}
/**
* Service Type
*
* S3 service type
*/
export enum S3Type {
S3 = "S3",
}
/**
* Source to get the .pbit files to extract lineage information
*
@ -3733,6 +3782,7 @@ export enum RESTType {
Snowflake = "Snowflake",
Spark = "Spark",
Spline = "Spline",
Ssis = "SSIS",
Stitch = "Stitch",
Superset = "Superset",
Synapse = "Synapse",

38
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts
View File

@ -2550,6 +2550,8 @@ export interface ServiceConnection {
*
* Wherescape Metadata Database Connection Config
*
* SSIS Metadata Database Connection Config
*
* Glue Pipeline Connection Config
*
* Airbyte Metadata Database Connection Config
@ -3837,6 +3839,10 @@ export interface ConfigClass {
* Underlying database connection
*/
databaseConnection?: DatabaseConnectionClass;
/**
* Underlying storage connection
*/
packageConnection?: S3Connection | string;
/**
* Fivetran API Secret.
*/
@ -5329,6 +5335,37 @@ export interface OracleConnectionType {
[property: string]: any;
}
/**
* S3 Connection.
*/
export interface S3Connection {
awsConfig: AWSCredentials;
/**
* Bucket Names of the data source.
*/
bucketNames?: string[];
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Regex to only fetch containers that matches the pattern.
*/
containerFilterPattern?: FilterPattern;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: S3Type;
}
/**
* Service Type
*
* S3 service type
*/
export enum S3Type {
S3 = "S3",
}
/**
* Source to get the .pbit files to extract lineage information
*
@ -5755,6 +5792,7 @@ export enum RESTType {
Snowflake = "Snowflake",
Spark = "Spark",
Spline = "Spline",
Ssis = "SSIS",
Stitch = "Stitch",
Superset = "Superset",
Synapse = "Synapse",

42
openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts
View File

@ -180,6 +180,8 @@ export interface PipelineConnection {
*
* Wherescape Metadata Database Connection Config
*
* SSIS Metadata Database Connection Config
*
* Glue Pipeline Connection Config
*
* Airbyte Metadata Database Connection Config
@ -257,7 +259,11 @@ export interface ConfigClass {
* Underlying database connection
*/
databaseConnection?: DatabaseConnectionClass;
awsConfig?: AWSCredentials;
/**
* Underlying storage connection
*/
packageConnection?: S3Connection | string;
awsConfig?: AWSCredentials;
/**
* Airbyte API version.
*/
@ -682,6 +688,8 @@ export interface AuthConfigurationType {
* Regex to only include/exclude schemas that matches the pattern.
*
* Regex to only include/exclude tables that matches the pattern.
*
* Regex to only fetch containers that matches the pattern.
*/
export interface FilterPattern {
/**
@ -955,6 +963,37 @@ export interface NifiCredentialsConfiguration {
clientkeyPath?: string;
}
/**
* S3 Connection.
*/
export interface S3Connection {
awsConfig: AWSCredentials;
/**
* Bucket Names of the data source.
*/
bucketNames?: string[];
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Regex to only fetch containers that matches the pattern.
*/
containerFilterPattern?: FilterPattern;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: S3Type;
}
/**
* Service Type
*
* S3 service type
*/
export enum S3Type {
S3 = "S3",
}
/**
* SASL Configuration details.
*
@ -1026,6 +1065,7 @@ export enum PipelineServiceType {
OpenLineage = "OpenLineage",
Spark = "Spark",
Spline = "Spline",
Ssis = "SSIS",
Stitch = "Stitch",
Wherescape = "Wherescape",
}

54
openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts
View File

@ -240,6 +240,8 @@ export interface ServiceConnection {
*
* Wherescape Metadata Database Connection Config
*
* SSIS Metadata Database Connection Config
*
* Glue Pipeline Connection Config
*
* Airbyte Metadata Database Connection Config
@ -1090,6 +1092,10 @@ export interface ConfigClass {
* Establish secure connection with Impala
*/
useSSL?: boolean;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
/**
* How to run the SQLite database. :memory: by default.
*/
@ -1523,6 +1529,10 @@ export interface ConfigClass {
* Underlying database connection
*/
databaseConnection?: DatabaseConnectionClass;
/**
* Underlying storage connection
*/
packageConnection?: S3Connection | string;
/**
* Fivetran API Secret.
*/
@ -1691,10 +1701,10 @@ export interface UsernamePasswordAuthentication {
*
* Regex exclude pipelines.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch containers that matches the pattern.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch search indexes that matches the pattern.
*/
export interface FilterPattern {
@ -2720,6 +2730,10 @@ export interface ConfigConnection {
* attempts to scan all the schemas.
*/
databaseSchema?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
/**
* HDB Store User Key generated from the command `hdbuserstore SET <KEY> <host:port>
* <USERNAME> <PASSWORD>`
@ -3220,6 +3234,10 @@ export interface HiveMetastoreConnectionDetails {
* attempts to scan all the schemas.
*/
databaseSchema?: string;
/**
* Use slow logs to extract lineage.
*/
useSlowLogs?: boolean;
}
/**
@ -3298,6 +3316,37 @@ export interface OracleConnectionType {
[property: string]: any;
}
/**
* S3 Connection.
*/
export interface S3Connection {
awsConfig: AWSCredentials;
/**
* Bucket Names of the data source.
*/
bucketNames?: string[];
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Regex to only fetch containers that matches the pattern.
*/
containerFilterPattern?: FilterPattern;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: S3Type;
}
/**
* Service Type
*
* S3 service type
*/
export enum S3Type {
S3 = "S3",
}
/**
* Source to get the .pbit files to extract lineage information
*
@ -3777,6 +3826,7 @@ export enum RESTType {
Snowflake = "Snowflake",
Spark = "Spark",
Spline = "Spline",
Ssis = "SSIS",
Stitch = "Stitch",
Superset = "Superset",
Synapse = "Synapse",

42
openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts
View File

@ -276,6 +276,8 @@ export interface ServiceConnection {
*
* Wherescape Metadata Database Connection Config
*
* SSIS Metadata Database Connection Config
*
* Glue Pipeline Connection Config
*
* Airbyte Metadata Database Connection Config
@ -1563,6 +1565,10 @@ export interface ConfigClass {
* Underlying database connection
*/
databaseConnection?: DatabaseConnectionClass;
/**
* Underlying storage connection
*/
packageConnection?: S3Connection | string;
/**
* Fivetran API Secret.
*/
@ -1731,10 +1737,10 @@ export interface UsernamePasswordAuthentication {
*
* Regex exclude pipelines.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch containers that matches the pattern.
*
* Regex to only fetch MlModels with names matching the pattern.
*
* Regex to only fetch search indexes that matches the pattern.
*
* Regex to only fetch databases that matches the pattern.
@ -3355,6 +3361,37 @@ export interface OracleConnectionType {
[property: string]: any;
}
/**
* S3 Connection.
*/
export interface S3Connection {
awsConfig: AWSCredentials;
/**
* Bucket Names of the data source.
*/
bucketNames?: string[];
connectionArguments?: { [key: string]: any };
connectionOptions?: { [key: string]: string };
/**
* Regex to only fetch containers that matches the pattern.
*/
containerFilterPattern?: FilterPattern;
supportsMetadataExtraction?: boolean;
/**
* Service Type
*/
type?: S3Type;
}
/**
* Service Type
*
* S3 service type
*/
export enum S3Type {
S3 = "S3",
}
/**
* Source to get the .pbit files to extract lineage information
*
@ -3834,6 +3871,7 @@ export enum RESTType {
Snowflake = "Snowflake",
Spark = "Spark",
Spline = "Spline",
Ssis = "SSIS",
Stitch = "Stitch",
Superset = "Superset",
Synapse = "Synapse",