mirror of
https://github.com/datahub-project/datahub.git
synced 2025-08-23 00:28:03 +00:00
105 lines
4.0 KiB
YAML
105 lines
4.0 KiB
YAML
---
|
|
services:
|
|
connect:
|
|
image: confluentinc/cp-kafka-connect:7.4.0
|
|
env_file: ./../kafka-connect/setup/connect.env
|
|
container_name: test_connect
|
|
hostname: test_connect
|
|
depends_on:
|
|
- zookeeper
|
|
- broker
|
|
- mysqldb
|
|
- mongo
|
|
ports:
|
|
- "28083:28083"
|
|
# volumes:
|
|
# - ./../kafka-connect/setup/confluentinc-kafka-connect-jdbc-10.2.5:/usr/local/share/kafka/plugins/confluentinc-kafka-connect-jdbc-10.2.5
|
|
# - ./../kafka-connect/setup/confluentinc-connect-transforms-1.4.1:/usr/local/share/kafka/plugins/confluentinc-connect-transforms-1.4.1
|
|
# - ./../kafka-connect/setup/debezium-debezium-connector-mysql-1.7.0:/usr/local/share/kafka/plugins/debezium-debezium-connector-mysql-1.7.0
|
|
# - ./../kafka-connect/setup/gcp-bigquery-project-keyfile.json:/usr/local/share/gcp-bigquery-project-keyfile.json
|
|
command:
|
|
- bash
|
|
- -c
|
|
- |
|
|
echo "Install confluent CLI"
|
|
# https://docs.confluent.io/confluent-cli/current/install.html#tarball-or-zip-installation
|
|
wget https://packages.confluent.io/confluent-cli/archives/4.16.0/confluent_linux_amd64.tar.gz
|
|
mkdir -p /tmp/confluent-cli
|
|
tar -xvf confluent_linux_amd64.tar.gz -C /tmp/confluent-cli
|
|
export PATH=/tmp/confluent-cli/confluent:$PATH
|
|
echo "Done!"
|
|
echo "Installing Connectors"
|
|
#
|
|
confluent connect plugin install --force confluentinc/kafka-connect-jdbc:10.2.5
|
|
#
|
|
confluent connect plugin install --force confluentinc/connect-transforms:1.4.1
|
|
#
|
|
confluent connect plugin install --force confluentinc/kafka-connect-datagen:0.6.0
|
|
#
|
|
confluent connect plugin install --force debezium/debezium-connector-mysql:1.7.0
|
|
#
|
|
confluent connect plugin install --force wepay/kafka-connect-bigquery:1.6.8
|
|
#
|
|
confluent connect plugin install --force mongodb/kafka-connect-mongodb:1.10.1
|
|
#
|
|
confluent connect plugin install --force confluentinc/kafka-connect-s3:10.5.1
|
|
#
|
|
mkdir -p /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib
|
|
#
|
|
curl -k -SL "https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.27.tar.gz" \
|
|
| tar -xzf - -C /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib \
|
|
--strip-components=1 mysql-connector-java-8.0.27/mysql-connector-java-8.0.27.jar
|
|
curl -k -SL "https://repo1.maven.org/maven2/io/strimzi/kafka-env-var-config-provider/0.1.1/kafka-env-var-config-provider-0.1.1.tar.gz" \
|
|
| tar -xzf - -C /usr/share/confluent-hub-components/
|
|
#
|
|
echo "Launching Kafka Connect worker"
|
|
#
|
|
/etc/confluent/docker/run &
|
|
#
|
|
sleep infinity
|
|
mysqldb:
|
|
image: mysql:8.0
|
|
environment:
|
|
MYSQL_ROOT_PASSWORD: rootpwd
|
|
MYSQL_USER: foo
|
|
MYSQL_PASSWORD: datahub
|
|
MYSQL_DATABASE: librarydb
|
|
container_name: test_mysql
|
|
hostname: test_mysql
|
|
ports:
|
|
- "23306:3306"
|
|
volumes:
|
|
- ./../kafka-connect/setup/conf/mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf
|
|
- ./../kafka-connect/setup/mysql-setup.sql:/docker-entrypoint-initdb.d/mysql-setup.sql
|
|
|
|
postgresdb:
|
|
image: postgres:alpine
|
|
container_name: "test_postgres"
|
|
environment:
|
|
POSTGRES_PASSWORD: datahub
|
|
volumes:
|
|
- ./../kafka-connect/setup/mysql-setup.sql:/docker-entrypoint-initdb.d/postgres_setup.sql
|
|
ports:
|
|
- "5432:5432"
|
|
|
|
mongo:
|
|
hostname: mongo
|
|
image: mongo:6.0.7
|
|
container_name: "test_mongo"
|
|
ports:
|
|
- "27017:27017"
|
|
command: --replSet rs0
|
|
environment:
|
|
# Don't set mongo admin user or password to avoid KeyFile authentication
|
|
# which is required when running MongoDB as a replica set
|
|
- MONGO_INITDB_DATABASE=test_db
|
|
volumes:
|
|
- ./../kafka-connect/setup/conf/:/scripts/
|
|
|
|
s3mock:
|
|
image: adobe/s3mock:2.13.0
|
|
environment:
|
|
- initialBuckets=test-bucket
|
|
ports:
|
|
- "9090:9090"
|