mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-10 17:46:02 +00:00
61 lines
1.6 KiB
YAML
61 lines
1.6 KiB
YAML
source:
|
|
type: databricks
|
|
config:
|
|
workspace_url: https://my-workspace.cloud.databricks.com
|
|
|
|
# Authentication Option 1: Personal Access Token
|
|
token: "<token>"
|
|
|
|
# Authentication Option 2: Azure Authentication (for Azure Databricks)
|
|
# Uncomment the following section and comment out the token above to use Azure auth
|
|
# azure_auth:
|
|
# client_id: "<azure_client_id>"
|
|
# tenant_id: "<azure_tenant_id>"
|
|
# client_secret: "<azure_client_secret>"
|
|
include_metastore: false
|
|
include_ownership: true
|
|
include_ml_model_aliases: false
|
|
ml_model_max_results: 1000
|
|
profiling:
|
|
method: "ge"
|
|
enabled: true
|
|
warehouse_id: "<warehouse_id>"
|
|
profile_table_level_only: false
|
|
max_wait_secs: 60
|
|
pattern:
|
|
deny:
|
|
- ".*\\.unwanted_schema"
|
|
|
|
# emit_siblings: true
|
|
# delta_lake_options:
|
|
# platform_instance_name: null
|
|
# env: 'PROD'
|
|
|
|
# profiling:
|
|
# method: "analyze"
|
|
# enabled: true
|
|
# warehouse_id: "<warehouse_id>"
|
|
# profile_table_level_only: true
|
|
# call_analyze: true
|
|
|
|
# catalogs: ["my_catalog"]
|
|
# schema_pattern:
|
|
# deny:
|
|
# - information_schema
|
|
# table_pattern:
|
|
# allow:
|
|
# - my_catalog.my_schema.my_table
|
|
# First you have to create domains on Datahub by following this guide -> https://docs.datahub.com/docs/domains/#domains-setup-prerequisites-and-permissions
|
|
# domain:
|
|
# urn:li:domain:1111-222-333-444-555:
|
|
# allow:
|
|
# - main.*
|
|
|
|
stateful_ingestion:
|
|
enabled: true
|
|
|
|
pipeline_name: acme-corp-unity
|
|
|
|
|
|
# sink configs if needed
|