mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-10-13 17:58:36 +00:00
Enhance documentation: Add required field annotations and clarify authentication details for various connectors in YAML configuration files. (#22054)
This commit is contained in:
Ayush Shah
GitHub
parent
134b4926cc
commit
d400da478f
@ -133,13 +133,13 @@ source:
|
||||
type: Looker
|
||||
```
|
||||
```yaml {% srNumber=1 %}
|
||||
clientId: Client ID
|
||||
clientId: Client ID # REQUIRED - Looker API client ID
|
||||
```
|
||||
```yaml {% srNumber=2 %}
|
||||
clientSecret: Client Secret
|
||||
clientSecret: Client Secret # REQUIRED - Looker API client secret
|
||||
```
|
||||
```yaml {% srNumber=3 %}
|
||||
hostPort: http://hostPort
|
||||
hostPort: http://hostPort # REQUIRED - Looker instance URL
|
||||
```
|
||||
```yaml {% srNumber=4 %}
|
||||
gitCredentials:
|
||||
|
||||
@ -251,13 +251,13 @@ source:
|
||||
type: PowerBI
|
||||
```
|
||||
```yaml {% srNumber=1 %}
|
||||
clientId: clientId
|
||||
clientId: clientId # REQUIRED - Azure AD application client ID
|
||||
```
|
||||
```yaml {% srNumber=2 %}
|
||||
clientSecret: secret
|
||||
clientSecret: secret # REQUIRED - Azure AD application client secret
|
||||
```
|
||||
```yaml {% srNumber=3 %}
|
||||
tenantId: tenant
|
||||
tenantId: tenant # REQUIRED - Azure AD tenant ID
|
||||
```
|
||||
```yaml {% srNumber=4 %}
|
||||
# scope:
|
||||
|
||||
@ -168,7 +168,7 @@ source:
|
||||
# personalAccessTokenSecret: personal_access_token_secret
|
||||
```
|
||||
```yaml {% srNumber=4 %}
|
||||
hostPort: http://localhost
|
||||
hostPort: http://localhost # REQUIRED - Tableau Server URL
|
||||
```
|
||||
```yaml {% srNumber=5 %}
|
||||
siteName: site_name
|
||||
|
||||
@ -154,8 +154,23 @@ This setting does **not** affect actual billing—it is only used for internal r
|
||||
The default value, if not set, may assume the standard on-demand BigQuery pricing (e.g., $5.00 per TiB), but you should adjust it according to your organization's negotiated rates or flat-rate pricing model.
|
||||
|
||||
{% note %}
|
||||
If you want to use [ADC authentication](https://cloud.google.com/docs/authentication#adc) for BigQuery you can just leave
|
||||
the GCP credentials empty. This is why they are not marked as required.
|
||||
**Application Default Credentials (ADC) Authentication**
|
||||
|
||||
If you want to use [ADC authentication](https://cloud.google.com/docs/authentication#adc) for BigQuery, configure the GCP credentials with type `gcp_adc`:
|
||||
|
||||
```yaml
|
||||
credentials:
|
||||
gcpConfig:
|
||||
type: gcp_adc
|
||||
projectId: ["your-project-id"] # Optional: specify project(s) for data access
|
||||
```
|
||||
|
||||
**Using ADC with Billing Project ID**: When using ADC authentication, you can still specify a **Billing Project ID** to ensure proper billing attribution for your BigQuery queries. This is particularly useful when:
|
||||
- Your service account has access to multiple projects
|
||||
- You want to bill queries to a specific project different from the one containing your data
|
||||
- You're running queries that span multiple projects
|
||||
|
||||
**ADC Setup**: ADC authentication works automatically when running in Google Cloud environments (GKE, Compute Engine, Cloud Run) or when you've configured it locally using `gcloud auth application-default login`.
|
||||
{% /note %}
|
||||
|
||||
|
||||
|
||||
@ -149,11 +149,26 @@ credentials:
|
||||
the GCP credentials empty. This is why they are not marked as required.
|
||||
|
||||
```yaml
|
||||
# ADC authentication with optional billing project
|
||||
...
|
||||
config:
|
||||
type: BigQuery
|
||||
credentials:
|
||||
gcpConfig: {}
|
||||
gcpConfig:
|
||||
type: gcp_adc
|
||||
projectId: ["your-data-project-id"] # Optional: specify project(s) for data access
|
||||
billingProjectId: "your-billing-project-id" # Optional: for billing attribution
|
||||
...
|
||||
```
|
||||
|
||||
```yaml
|
||||
# ADC authentication without billing project
|
||||
...
|
||||
config:
|
||||
type: BigQuery
|
||||
credentials:
|
||||
gcpConfig:
|
||||
type: gcp_adc
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
@ -160,7 +160,7 @@ source:
|
||||
username: <username>
|
||||
password: <password>
|
||||
hostPort: <hostPort>
|
||||
# database: <database>
|
||||
database: <database> # REQUIRED - database name
|
||||
```
|
||||
```yaml {% srNumber=8 %}
|
||||
# connectionOptions:
|
||||
|
||||
@ -444,31 +444,37 @@ source:
|
||||
type: Mysql
|
||||
```
|
||||
```yaml {% srNumber=1 %}
|
||||
username: <username>
|
||||
username: <username> # REQUIRED
|
||||
```
|
||||
```yaml {% srNumber=2 %}
|
||||
authType:
|
||||
password: <password>
|
||||
password: <password> # Basic Auth - most common
|
||||
```
|
||||
```yaml {% srNumber=3 %}
|
||||
authType:
|
||||
awsConfig:
|
||||
awsConfig: # IAM Auth for AWS RDS MySQL
|
||||
awsAccessKeyId: access key id
|
||||
awsSecretAccessKey: access secret key
|
||||
awsRegion: aws region name
|
||||
```
|
||||
```yaml {% srNumber=4 %}
|
||||
hostPort: <hostPort>
|
||||
hostPort: <hostPort> # REQUIRED - format: host:port
|
||||
```
|
||||
```yaml {% srNumber=5 %}
|
||||
databaseSchema: schema
|
||||
databaseSchema: schema # Optional: restrict to single schema
|
||||
```
|
||||
```yaml {% srNumber=6 %}
|
||||
databaseName: database_name # Optional: custom name in OpenMetadata
|
||||
```
|
||||
```yaml {% srNumber=7 %}
|
||||
useSlowLogs: false # Optional: use slow logs for lineage (default: false)
|
||||
```
|
||||
|
||||
```yaml {% srNumber=6 %}
|
||||
```yaml {% srNumber=8 %}
|
||||
# connectionOptions:
|
||||
# key: value
|
||||
```
|
||||
```yaml {% srNumber=7 %}
|
||||
```yaml {% srNumber=9 %}
|
||||
# connectionArguments:
|
||||
# key: value
|
||||
```
|
||||
|
||||
@ -282,24 +282,24 @@ source:
|
||||
type: PostgreSQL
|
||||
```
|
||||
```yaml {% srNumber=1 %}
|
||||
username: username
|
||||
username: username # REQUIRED
|
||||
```
|
||||
```yaml {% srNumber=2 %}
|
||||
authType:
|
||||
password: <password>
|
||||
password: <password> # Basic Auth - most common
|
||||
```
|
||||
```yaml {% srNumber=3 %}
|
||||
authType:
|
||||
awsConfig:
|
||||
awsConfig: # IAM Auth for AWS RDS PostgreSQL
|
||||
awsAccessKeyId: access key id
|
||||
awsSecretAccessKey: access secret key
|
||||
awsRegion: aws region name
|
||||
```
|
||||
```yaml {% srNumber=4 %}
|
||||
hostPort: localhost:5432
|
||||
hostPort: localhost:5432 # REQUIRED - format: host:port
|
||||
```
|
||||
```yaml {% srNumber=5 %}
|
||||
database: database
|
||||
database: database # REQUIRED - database name
|
||||
```
|
||||
```yaml {% srNumber=6 %}
|
||||
ingestAllDatabases: true
|
||||
|
||||
@ -81,7 +81,7 @@ This is a sample config for Redshift:
|
||||
|
||||
{% codeInfo srNumber=1 %}
|
||||
|
||||
**username**: Specify the User to connect to Snoflake. It should have enough privileges to read all the metadata.
|
||||
**username**: Specify the User to connect to Redshift. It should have enough privileges to read all the metadata.
|
||||
|
||||
{% /codeInfo %}
|
||||
|
||||
@ -166,16 +166,16 @@ source:
|
||||
type: Redshift
|
||||
```
|
||||
```yaml {% srNumber=1 %}
|
||||
username: username
|
||||
username: username # REQUIRED
|
||||
```
|
||||
```yaml {% srNumber=2 %}
|
||||
password: password
|
||||
```
|
||||
```yaml {% srNumber=3 %}
|
||||
database: dev
|
||||
database: dev # REQUIRED - database name
|
||||
```
|
||||
```yaml {% srNumber=4 %}
|
||||
hostPort: cluster.name.region.redshift.amazonaws.com:5439
|
||||
hostPort: cluster.name.region.redshift.amazonaws.com:5439 # REQUIRED - format: host:port
|
||||
```
|
||||
```yaml {% srNumber=5 %}
|
||||
# ingestAllDatabases: true
|
||||
|
||||
@ -157,39 +157,57 @@ source:
|
||||
type: Kafka
|
||||
```
|
||||
```yaml {% srNumber=1 %}
|
||||
bootstrapServers: localhost:9092
|
||||
bootstrapServers: localhost:9092 # REQUIRED - Kafka broker addresses
|
||||
```
|
||||
```yaml {% srNumber=2 %}
|
||||
schemaRegistryURL: http://localhost:8081 # Needs to be a URI
|
||||
schemaRegistryURL: http://localhost:8081 # Schema Registry URL
|
||||
```
|
||||
```yaml {% srNumber=3 %}
|
||||
saslUsername: username
|
||||
saslUsername: username # SASL authentication username
|
||||
```
|
||||
```yaml {% srNumber=4 %}
|
||||
saslPassword: password
|
||||
saslPassword: password # SASL authentication password
|
||||
```
|
||||
```yaml {% srNumber=5 %}
|
||||
saslMechanism: PLAIN
|
||||
saslMechanism: PLAIN # SASL mechanism: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, GSSAPI, OAUTHBEARER
|
||||
```
|
||||
```yaml {% srNumber=6 %}
|
||||
basicAuthUserInfo: username:password
|
||||
basicAuthUserInfo: username:password # Schema Registry basic auth
|
||||
```
|
||||
```yaml {% srNumber=7 %}
|
||||
consumerConfig: {}
|
||||
consumerConfig:
|
||||
# Example consumer configurations:
|
||||
# auto.offset.reset: "earliest"
|
||||
# max.poll.records: 500
|
||||
# session.timeout.ms: 30000
|
||||
```
|
||||
```yaml {% srNumber=8 %}
|
||||
schemaRegistryConfig: {}
|
||||
schemaRegistryConfig:
|
||||
# Example schema registry configurations:
|
||||
# schema.registry.basic.auth.credentials.source: "USER_INFO"
|
||||
# schema.registry.ssl.truststore.location: "/path/to/truststore.jks"
|
||||
```
|
||||
```yaml {% srNumber=9 %}
|
||||
# securityProtocol: PLAINTEXT
|
||||
securityProtocol: PLAINTEXT # Security protocol: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL
|
||||
```
|
||||
```yaml {% srNumber=10 %}
|
||||
# schemaRegistryTopicSuffixName: -value
|
||||
```
|
||||
```yaml {% srNumber=11 %}
|
||||
# schemaRegistrySSL: ""
|
||||
# SSL Configuration for Consumer connections (when securityProtocol is SSL or SASL_SSL)
|
||||
# consumerConfigSSL:
|
||||
# caCertificate: "/path/to/ca-cert.pem"
|
||||
# sslCertificate: "/path/to/client-cert.pem"
|
||||
# sslKey: "/path/to/client-key.pem"
|
||||
```
|
||||
```yaml {% srNumber=12 %}
|
||||
# SSL Configuration for Schema Registry connections
|
||||
# schemaRegistrySSL:
|
||||
# caCertificate: "/path/to/schema-registry-ca.pem"
|
||||
# sslCertificate: "/path/to/schema-registry-cert.pem"
|
||||
# sslKey: "/path/to/schema-registry-key.pem"
|
||||
```
|
||||
```yaml {% srNumber=13 %}
|
||||
# supportsMetadataExtraction: true
|
||||
```
|
||||
|
||||
|
||||
@ -95,7 +95,7 @@ source:
|
||||
type: Dagster
|
||||
```
|
||||
```yaml {% srNumber=1 %}
|
||||
host: "https://<yourorghere>.dagster.cloud/prod" # or http://127.0.0.1:3000
|
||||
host: "https://<yourorghere>.dagster.cloud/prod" # REQUIRED - Dagster instance URL
|
||||
```
|
||||
```yaml {% srNumber=2 %}
|
||||
token: token
|
||||
|
||||
@ -138,12 +138,12 @@ source:
|
||||
type: ElasticSearch
|
||||
```
|
||||
```yaml {% srNumber=1 %}
|
||||
hostPort: http://localhost:9200
|
||||
hostPort: http://localhost:9200 # REQUIRED - Elasticsearch cluster URL
|
||||
```
|
||||
```yaml {% srNumber=2 %}
|
||||
authType:
|
||||
username: elastic
|
||||
password: my_own_password
|
||||
username: elastic # Basic auth username
|
||||
password: my_own_password # Basic auth password
|
||||
```
|
||||
```yaml {% srNumber=3 %}
|
||||
# apiKeyId: <api key id>
|
||||
|
||||
@ -167,7 +167,7 @@ source:
|
||||
serviceConnection:
|
||||
config:
|
||||
type: S3
|
||||
awsConfig:
|
||||
awsConfig: # REQUIRED - AWS authentication configuration
|
||||
```
|
||||
|
||||
{% partial file="/v1.9/connectors/yaml/common/aws-config.md" /%}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user