2023-03-15 02:29:54 +05:30
|
|
|
---
|
|
|
|
|
run_id: kafka-connect-run
|
|
|
|
|
|
2025-04-28 23:34:33 +09:00
|
|
|
# see https://docs.datahub.com/docs/generated/ingestion/sources/kafka-connect for complete documentation
|
2023-03-15 02:29:54 +05:30
|
|
|
source:
|
|
|
|
|
type: "kafka-connect"
|
|
|
|
|
config:
|
2023-04-04 18:38:30 +05:30
|
|
|
connect_uri: "http://localhost:28083"
|
2023-03-15 02:29:54 +05:30
|
|
|
connector_patterns:
|
|
|
|
|
allow:
|
|
|
|
|
- source_mongodb_connector
|
|
|
|
|
provided_configs:
|
|
|
|
|
- provider: env
|
|
|
|
|
path_key: MYSQL_CONNECTION_URL
|
|
|
|
|
value: jdbc:mysql://test_mysql:3306/librarydb
|
|
|
|
|
- provider: env
|
|
|
|
|
path_key: POSTGRES_CONNECTION_URL
|
|
|
|
|
value: jdbc:postgresql://test_postgres:5432/postgres
|
|
|
|
|
convert_lineage_urns_to_lowercase: true
|
|
|
|
|
platform_instance_map: # optional
|
|
|
|
|
mysql: mysql1 # optional
|
2023-07-13 16:00:42 +05:30
|
|
|
kafka: kafkax
|
2023-03-15 02:29:54 +05:30
|
|
|
connect_to_platform_map: # optional
|
|
|
|
|
postgres_source: # optional - Connector name
|
|
|
|
|
postgres: postgres1 # optional - Platform to instance map
|
|
|
|
|
generic_connectors: # optional - Generic connector lineage info
|
|
|
|
|
- connector_name: generic-connector
|
|
|
|
|
source_dataset: generic-dataset
|
|
|
|
|
source_platform: generic-platform
|
|
|
|
|
|
2025-04-28 23:34:33 +09:00
|
|
|
# see https://docs.datahub.com/docs/metadata-ingestion/sink_docs/datahub for complete documentation
|
2023-03-15 02:29:54 +05:30
|
|
|
sink:
|
|
|
|
|
type: file
|
|
|
|
|
config:
|
|
|
|
|
filename: "./kafka_connect_mces.json"
|
