mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-03 23:28:11 +00:00
50 lines
1.2 KiB
YAML
50 lines
1.2 KiB
YAML
source:
|
|
type: unity-catalog
|
|
config:
|
|
workspace_url: https://my-workspace.cloud.databricks.com
|
|
token: "<token>"
|
|
include_metastore: false
|
|
include_ownership: true
|
|
profiling:
|
|
method: "ge"
|
|
enabled: true
|
|
warehouse_id: "<warehouse_id>"
|
|
profile_table_level_only: false
|
|
max_wait_secs: 60
|
|
pattern:
|
|
deny:
|
|
- ".*\\.unwanted_schema"
|
|
|
|
# emit_siblings: true
|
|
# delta_lake_options:
|
|
# platform_instance_name: null
|
|
# env: 'PROD'
|
|
|
|
# profiling:
|
|
# method: "analyze"
|
|
# enabled: true
|
|
# warehouse_id: "<warehouse_id>"
|
|
# profile_table_level_only: true
|
|
# call_analyze: true
|
|
|
|
# catalogs: ["my_catalog"]
|
|
# schema_pattern:
|
|
# deny:
|
|
# - information_schema
|
|
# table_pattern:
|
|
# allow:
|
|
# - my_catalog.my_schema.my_table
|
|
# First you have to create domains on Datahub by following this guide -> https://docs.datahub.com/docs/domains/#domains-setup-prerequisites-and-permissions
|
|
# domain:
|
|
# urn:li:domain:1111-222-333-444-555:
|
|
# allow:
|
|
# - main.*
|
|
|
|
stateful_ingestion:
|
|
enabled: true
|
|
|
|
pipeline_name: acme-corp-unity
|
|
|
|
|
|
# sink configs if needed
|