From d400da478fc4921e93d78e86fec0459c0c9d6e56 Mon Sep 17 00:00:00 2001 From: Ayush Shah Date: Tue, 1 Jul 2025 13:58:16 +0530 Subject: [PATCH] Enhance documentation: Add required field annotations and clarify authentication details for various connectors in YAML configuration files. (#22054) --- .../connectors/dashboard/looker/yaml.md | 6 +-- .../connectors/dashboard/powerbi/yaml.md | 6 +-- .../connectors/dashboard/tableau/yaml.md | 2 +- .../connectors/database/bigquery/index.md | 19 +++++++++- .../connectors/database/bigquery/yaml.md | 17 ++++++++- .../connectors/database/mssql/yaml.md | 2 +- .../connectors/database/mysql/yaml.md | 24 +++++++----- .../connectors/database/postgres/yaml.md | 20 +++++----- .../connectors/database/redshift/yaml.md | 8 ++-- .../connectors/messaging/kafka/yaml.md | 38 ++++++++++++++----- .../connectors/pipeline/dagster/yaml.md | 2 +- .../connectors/search/elasticsearch/yaml.md | 6 +-- .../connectors/storage/s3/yaml.md | 2 +- 13 files changed, 103 insertions(+), 49 deletions(-) diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/looker/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/looker/yaml.md index 6e51c0e0484..9f67a79d3e0 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/looker/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/looker/yaml.md @@ -133,13 +133,13 @@ source: type: Looker ``` ```yaml {% srNumber=1 %} - clientId: Client ID + clientId: Client ID # REQUIRED - Looker API client ID ``` ```yaml {% srNumber=2 %} - clientSecret: Client Secret + clientSecret: Client Secret # REQUIRED - Looker API client secret ``` ```yaml {% srNumber=3 %} - hostPort: http://hostPort + hostPort: http://hostPort # REQUIRED - Looker instance URL ``` ```yaml {% srNumber=4 %} gitCredentials: diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/powerbi/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/powerbi/yaml.md index 9d19fb7eb8e..fd6e7f02a76 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/powerbi/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/powerbi/yaml.md @@ -251,13 +251,13 @@ source: type: PowerBI ``` ```yaml {% srNumber=1 %} - clientId: clientId + clientId: clientId # REQUIRED - Azure AD application client ID ``` ```yaml {% srNumber=2 %} - clientSecret: secret + clientSecret: secret # REQUIRED - Azure AD application client secret ``` ```yaml {% srNumber=3 %} - tenantId: tenant + tenantId: tenant # REQUIRED - Azure AD tenant ID ``` ```yaml {% srNumber=4 %} # scope: diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/tableau/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/tableau/yaml.md index dab4187b4a1..2de4de61e04 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/tableau/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/dashboard/tableau/yaml.md @@ -168,7 +168,7 @@ source: # personalAccessTokenSecret: personal_access_token_secret ``` ```yaml {% srNumber=4 %} - hostPort: http://localhost + hostPort: http://localhost # REQUIRED - Tableau Server URL ``` ```yaml {% srNumber=5 %} siteName: site_name diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/bigquery/index.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/bigquery/index.md index b909934c20f..42a214e2bb2 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/bigquery/index.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/bigquery/index.md @@ -154,8 +154,23 @@ This setting does **not** affect actual billing—it is only used for internal r The default value, if not set, may assume the standard on-demand BigQuery pricing (e.g., $5.00 per TiB), but you should adjust it according to your organization's negotiated rates or flat-rate pricing model. {% note %} -If you want to use [ADC authentication](https://cloud.google.com/docs/authentication#adc) for BigQuery you can just leave -the GCP credentials empty. This is why they are not marked as required. +**Application Default Credentials (ADC) Authentication** + +If you want to use [ADC authentication](https://cloud.google.com/docs/authentication#adc) for BigQuery, configure the GCP credentials with type `gcp_adc`: + +```yaml +credentials: + gcpConfig: + type: gcp_adc + projectId: ["your-project-id"] # Optional: specify project(s) for data access +``` + +**Using ADC with Billing Project ID**: When using ADC authentication, you can still specify a **Billing Project ID** to ensure proper billing attribution for your BigQuery queries. This is particularly useful when: +- Your service account has access to multiple projects +- You want to bill queries to a specific project different from the one containing your data +- You're running queries that span multiple projects + +**ADC Setup**: ADC authentication works automatically when running in Google Cloud environments (GKE, Compute Engine, Cloud Run) or when you've configured it locally using `gcloud auth application-default login`. {% /note %} diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/bigquery/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/bigquery/yaml.md index b3c93044310..076080b74e2 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/bigquery/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/bigquery/yaml.md @@ -149,11 +149,26 @@ credentials: the GCP credentials empty. This is why they are not marked as required. ```yaml +# ADC authentication with optional billing project ... config: type: BigQuery credentials: - gcpConfig: {} + gcpConfig: + type: gcp_adc + projectId: ["your-data-project-id"] # Optional: specify project(s) for data access + billingProjectId: "your-billing-project-id" # Optional: for billing attribution +... +``` + +```yaml +# ADC authentication without billing project +... + config: + type: BigQuery + credentials: + gcpConfig: + type: gcp_adc ... ``` diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/mssql/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/mssql/yaml.md index b5473218e71..3166633a2fb 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/mssql/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/mssql/yaml.md @@ -160,7 +160,7 @@ source: username: password: hostPort: - # database: + database: # REQUIRED - database name ``` ```yaml {% srNumber=8 %} # connectionOptions: diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/mysql/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/mysql/yaml.md index 5820d27341d..aab9b50d934 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/mysql/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/mysql/yaml.md @@ -444,31 +444,37 @@ source: type: Mysql ``` ```yaml {% srNumber=1 %} - username: + username: # REQUIRED ``` ```yaml {% srNumber=2 %} - authType: - password: + authType: + password: # Basic Auth - most common ``` ```yaml {% srNumber=3 %} - authType: - awsConfig: + authType: + awsConfig: # IAM Auth for AWS RDS MySQL awsAccessKeyId: access key id awsSecretAccessKey: access secret key awsRegion: aws region name ``` ```yaml {% srNumber=4 %} - hostPort: + hostPort: # REQUIRED - format: host:port ``` ```yaml {% srNumber=5 %} - databaseSchema: schema + databaseSchema: schema # Optional: restrict to single schema +``` +```yaml {% srNumber=6 %} + databaseName: database_name # Optional: custom name in OpenMetadata +``` +```yaml {% srNumber=7 %} + useSlowLogs: false # Optional: use slow logs for lineage (default: false) ``` -```yaml {% srNumber=6 %} +```yaml {% srNumber=8 %} # connectionOptions: # key: value ``` -```yaml {% srNumber=7 %} +```yaml {% srNumber=9 %} # connectionArguments: # key: value ``` diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/postgres/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/postgres/yaml.md index e6c1f799c92..b1ab835ee28 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/postgres/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/postgres/yaml.md @@ -282,24 +282,24 @@ source: type: PostgreSQL ``` ```yaml {% srNumber=1 %} - username: username + username: username # REQUIRED ``` ```yaml {% srNumber=2 %} - authType: - password: + authType: + password: # Basic Auth - most common ``` ```yaml {% srNumber=3 %} - authType: - awsConfig: - awsAccessKeyId: access key id - awsSecretAccessKey: access secret key - awsRegion: aws region name + authType: + awsConfig: # IAM Auth for AWS RDS PostgreSQL + awsAccessKeyId: access key id + awsSecretAccessKey: access secret key + awsRegion: aws region name ``` ```yaml {% srNumber=4 %} - hostPort: localhost:5432 + hostPort: localhost:5432 # REQUIRED - format: host:port ``` ```yaml {% srNumber=5 %} - database: database + database: database # REQUIRED - database name ``` ```yaml {% srNumber=6 %} ingestAllDatabases: true diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/redshift/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/redshift/yaml.md index 674d46d020c..48affdd624c 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/redshift/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/database/redshift/yaml.md @@ -81,7 +81,7 @@ This is a sample config for Redshift: {% codeInfo srNumber=1 %} -**username**: Specify the User to connect to Snoflake. It should have enough privileges to read all the metadata. +**username**: Specify the User to connect to Redshift. It should have enough privileges to read all the metadata. {% /codeInfo %} @@ -166,16 +166,16 @@ source: type: Redshift ``` ```yaml {% srNumber=1 %} - username: username + username: username # REQUIRED ``` ```yaml {% srNumber=2 %} password: password ``` ```yaml {% srNumber=3 %} - database: dev + database: dev # REQUIRED - database name ``` ```yaml {% srNumber=4 %} - hostPort: cluster.name.region.redshift.amazonaws.com:5439 + hostPort: cluster.name.region.redshift.amazonaws.com:5439 # REQUIRED - format: host:port ``` ```yaml {% srNumber=5 %} # ingestAllDatabases: true diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/messaging/kafka/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/messaging/kafka/yaml.md index 849d52a4364..9edcfd1ff39 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/messaging/kafka/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/messaging/kafka/yaml.md @@ -157,39 +157,57 @@ source: type: Kafka ``` ```yaml {% srNumber=1 %} - bootstrapServers: localhost:9092 + bootstrapServers: localhost:9092 # REQUIRED - Kafka broker addresses ``` ```yaml {% srNumber=2 %} - schemaRegistryURL: http://localhost:8081 # Needs to be a URI + schemaRegistryURL: http://localhost:8081 # Schema Registry URL ``` ```yaml {% srNumber=3 %} - saslUsername: username + saslUsername: username # SASL authentication username ``` ```yaml {% srNumber=4 %} - saslPassword: password + saslPassword: password # SASL authentication password ``` ```yaml {% srNumber=5 %} - saslMechanism: PLAIN + saslMechanism: PLAIN # SASL mechanism: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, GSSAPI, OAUTHBEARER ``` ```yaml {% srNumber=6 %} - basicAuthUserInfo: username:password + basicAuthUserInfo: username:password # Schema Registry basic auth ``` ```yaml {% srNumber=7 %} - consumerConfig: {} + consumerConfig: + # Example consumer configurations: + # auto.offset.reset: "earliest" + # max.poll.records: 500 + # session.timeout.ms: 30000 ``` ```yaml {% srNumber=8 %} - schemaRegistryConfig: {} + schemaRegistryConfig: + # Example schema registry configurations: + # schema.registry.basic.auth.credentials.source: "USER_INFO" + # schema.registry.ssl.truststore.location: "/path/to/truststore.jks" ``` ```yaml {% srNumber=9 %} - # securityProtocol: PLAINTEXT + securityProtocol: PLAINTEXT # Security protocol: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL ``` ```yaml {% srNumber=10 %} # schemaRegistryTopicSuffixName: -value ``` ```yaml {% srNumber=11 %} - # schemaRegistrySSL: "" + # SSL Configuration for Consumer connections (when securityProtocol is SSL or SASL_SSL) + # consumerConfigSSL: + # caCertificate: "/path/to/ca-cert.pem" + # sslCertificate: "/path/to/client-cert.pem" + # sslKey: "/path/to/client-key.pem" ``` ```yaml {% srNumber=12 %} + # SSL Configuration for Schema Registry connections + # schemaRegistrySSL: + # caCertificate: "/path/to/schema-registry-ca.pem" + # sslCertificate: "/path/to/schema-registry-cert.pem" + # sslKey: "/path/to/schema-registry-key.pem" +``` +```yaml {% srNumber=13 %} # supportsMetadataExtraction: true ``` diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/pipeline/dagster/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/pipeline/dagster/yaml.md index 07217fe3bd3..0bda6629eb3 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/pipeline/dagster/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/pipeline/dagster/yaml.md @@ -95,7 +95,7 @@ source: type: Dagster ``` ```yaml {% srNumber=1 %} - host: "https://.dagster.cloud/prod" # or http://127.0.0.1:3000 + host: "https://.dagster.cloud/prod" # REQUIRED - Dagster instance URL ``` ```yaml {% srNumber=2 %} token: token diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/search/elasticsearch/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/search/elasticsearch/yaml.md index 262509bd306..321973afdf0 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/search/elasticsearch/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/search/elasticsearch/yaml.md @@ -138,12 +138,12 @@ source: type: ElasticSearch ``` ```yaml {% srNumber=1 %} - hostPort: http://localhost:9200 + hostPort: http://localhost:9200 # REQUIRED - Elasticsearch cluster URL ``` ```yaml {% srNumber=2 %} authType: - username: elastic - password: my_own_password + username: elastic # Basic auth username + password: my_own_password # Basic auth password ``` ```yaml {% srNumber=3 %} # apiKeyId: diff --git a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/storage/s3/yaml.md b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/storage/s3/yaml.md index c59e476b8ee..f29fd7ec541 100644 --- a/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/storage/s3/yaml.md +++ b/openmetadata-docs/content/v1.9.x-SNAPSHOT/connectors/storage/s3/yaml.md @@ -167,7 +167,7 @@ source: serviceConnection: config: type: S3 - awsConfig: + awsConfig: # REQUIRED - AWS authentication configuration ``` {% partial file="/v1.9/connectors/yaml/common/aws-config.md" /%}