datahub/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml

---
services:
  connect:
    image: confluentinc/cp-kafka-connect:7.4.0
    env_file: ./../kafka-connect/setup/connect.env
    container_name: test_connect
    hostname: test_connect
    depends_on:
      - zookeeper
      - broker
      - mysqldb
      - mongo
    ports:
      - "28083:28083"
#    volumes:        
#      - ./../kafka-connect/setup/confluentinc-kafka-connect-jdbc-10.2.5:/usr/local/share/kafka/plugins/confluentinc-kafka-connect-jdbc-10.2.5
#      - ./../kafka-connect/setup/confluentinc-connect-transforms-1.4.1:/usr/local/share/kafka/plugins/confluentinc-connect-transforms-1.4.1
#      - ./../kafka-connect/setup/debezium-debezium-connector-mysql-1.7.0:/usr/local/share/kafka/plugins/debezium-debezium-connector-mysql-1.7.0
#      - ./../kafka-connect/setup/gcp-bigquery-project-keyfile.json:/usr/local/share/gcp-bigquery-project-keyfile.json
    command:
      - bash
      - -c
      - |
        echo "Install confluent CLI"
        # https://docs.confluent.io/confluent-cli/current/install.html#tarball-or-zip-installation
        wget https://packages.confluent.io/confluent-cli/archives/4.16.0/confluent_linux_amd64.tar.gz
        mkdir -p /tmp/confluent-cli
        tar -xvf confluent_linux_amd64.tar.gz -C /tmp/confluent-cli
        export PATH=/tmp/confluent-cli/confluent:$PATH
        echo "Done!"
        echo "Installing Connectors"
        #
        confluent connect plugin install --force confluentinc/kafka-connect-jdbc:10.2.5      
        #
        confluent connect plugin install --force confluentinc/connect-transforms:1.4.1
        #
        confluent connect plugin install --force confluentinc/kafka-connect-datagen:0.6.0
        #
        confluent connect plugin install --force debezium/debezium-connector-mysql:1.7.0
        #
        confluent connect plugin install --force wepay/kafka-connect-bigquery:1.6.8
        #
        confluent connect plugin install --force mongodb/kafka-connect-mongodb:1.10.1
        #
        confluent connect plugin install --force confluentinc/kafka-connect-s3:10.5.1
        #
        mkdir -p /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib
        #
        curl -k -SL "https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.27.tar.gz" \
             | tar -xzf - -C /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib \
             --strip-components=1 mysql-connector-java-8.0.27/mysql-connector-java-8.0.27.jar
        curl -k -SL "https://repo1.maven.org/maven2/io/strimzi/kafka-env-var-config-provider/0.1.1/kafka-env-var-config-provider-0.1.1.tar.gz" \
             | tar -xzf - -C /usr/share/confluent-hub-components/
        #             
        echo "Launching Kafka Connect worker"
        #
        /etc/confluent/docker/run &
        #
        sleep infinity
  mysqldb:
    image: mysql:8.0
    environment:
      MYSQL_ROOT_PASSWORD: rootpwd
      MYSQL_USER: foo
      MYSQL_PASSWORD: datahub
      MYSQL_DATABASE: librarydb
    container_name: test_mysql
    hostname: test_mysql
    ports:
      - "23306:3306"
    volumes:  
      - ./../kafka-connect/setup/conf/mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf
      - ./../kafka-connect/setup/mysql-setup.sql:/docker-entrypoint-initdb.d/mysql-setup.sql

  postgresdb:
    image: postgres:alpine
    container_name: "test_postgres"
    environment:
      POSTGRES_PASSWORD: datahub
    volumes:  
      - ./../kafka-connect/setup/mysql-setup.sql:/docker-entrypoint-initdb.d/postgres_setup.sql
    ports:
      - "5432:5432"

  mongo:
    hostname: mongo
    image: mongo:6.0.7
    container_name: "test_mongo"
    ports:
      - "27017:27017"
    command: --replSet rs0
    environment:
      # Don't set mongo admin user or password to avoid KeyFile authentication
      # which is required when running MongoDB as a replica set
      - MONGO_INITDB_DATABASE=test_db
    volumes:
      - ./../kafka-connect/setup/conf/:/scripts/

  s3mock:
    image: adobe/s3mock:2.13.0
    environment:
      - initialBuckets=test-bucket
    ports:
      - "9090:9090"