datahub/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml

---
services:
  connect:
    image: confluentinc/cp-kafka-connect:7.4.0
    platform: linux/amd64
    env_file: ./../kafka-connect/setup/connect.env
    container_name: test_connect
    hostname: test_connect
    depends_on:
      zookeeper:
        condition: service_healthy
      broker:
        condition: service_healthy
      mysqldb:
        condition: service_healthy
      postgresdb:
        condition: service_healthy
      sqlserverdb:
        condition: service_healthy
      mongo:
        condition: service_healthy
    ports:
      - "28083:28083"
    restart: on-failure:3
    deploy:
      resources:
        limits:
          memory: 768M
        reservations:
          memory: 384M
    command:
      - bash
      - -c
      - |
        echo "Installing connectors using confluent-hub"
        
        # First verify that confluent-hub exists
        if ! [ -x "$(command -v confluent-hub)" ]; then
          echo "Error: confluent-hub is not installed." >&2
          exit 1
        fi
        
        echo "confluent-hub installation verified"
        echo "Installing connectors..."
        
        # Install with confluent-hub - ensure all connectors install correctly
        confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.2.5 || { echo "Failed to install JDBC connector"; exit 1; }
        confluent-hub install --no-prompt confluentinc/connect-transforms:1.4.1 || { echo "Failed to install transforms"; exit 1; }
        confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.6.0 || { echo "Failed to install datagen"; exit 1; }
        confluent-hub install --no-prompt debezium/debezium-connector-mysql:2.4.2 || { echo "Failed to install Debezium MySQL"; exit 1; }
        confluent-hub install --no-prompt debezium/debezium-connector-postgresql:2.5.4 || { echo "Failed to install Debezium PostgreSQL"; exit 1; }
        confluent-hub install --no-prompt debezium/debezium-connector-sqlserver:2.5.4 || { echo "Failed to install Debezium SQL Server"; exit 1; }
        confluent-hub install --no-prompt wepay/kafka-connect-bigquery:2.5.6 || { echo "Failed to install BigQuery"; exit 1; }
        confluent-hub install --no-prompt mongodb/kafka-connect-mongodb:1.10.1 || { echo "Failed to install MongoDB"; exit 1; }
        confluent-hub install --no-prompt confluentinc/kafka-connect-s3:10.5.1 || { echo "Failed to install S3"; exit 1; }
        
        echo "All connectors installed successfully"
        
        # Install MySQL JDBC Driver
        mkdir -p /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib
        echo "Installing MySQL JDBC Driver..."
        curl -k -SL "https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.27.tar.gz" \
             | tar -xzf - -C /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib \
             --strip-components=1 mysql-connector-java-8.0.27/mysql-connector-java-8.0.27.jar \
             || { echo "Failed to install MySQL JDBC driver"; exit 1; }
        
        # Install EnvVar Config Provider
        echo "Installing EnvVar Config Provider..."
        curl -k -SL "https://repo1.maven.org/maven2/io/strimzi/kafka-env-var-config-provider/0.1.1/kafka-env-var-config-provider-0.1.1.tar.gz" \
             | tar -xzf - -C /usr/share/confluent-hub-components/ \
             || { echo "Failed to install EnvVar Config Provider"; exit 1; }
             
        echo "All components installed successfully, launching Kafka Connect worker..."
        
        # Create a flag file to indicate installation is complete for the healthcheck
        # However, this results on a healtcheck not very trustable, as the service startup just after this takes a significant amount of time
        # So client validation is needed; see check_connectors_ready
        touch /tmp/connectors_installed
        
        # Launch connect worker and ensure it's in the foreground to catch any startup errors
        exec /etc/confluent/docker/run
    healthcheck:
      test: ["CMD-SHELL", "test -f /tmp/connectors_installed || (echo 'Not all plugins installed yet' && exit 1)"]
      interval: 10s
      timeout: 10s
      retries: 20
      start_period: 60s
  mysqldb:
    image: mysql:8.0
    environment:
      MYSQL_ROOT_PASSWORD: rootpwd
      MYSQL_USER: foo
      MYSQL_PASSWORD: datahub
      MYSQL_DATABASE: librarydb
    container_name: test_mysql
    hostname: test_mysql
    ports:
      - "23306:3306"
    volumes:  
      - ./../kafka-connect/setup/conf/mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf
      - ./../kafka-connect/setup/mysql-setup.sql:/docker-entrypoint-initdb.d/mysql-setup.sql
    healthcheck:
      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-prootpwd"]
      interval: 5s
      timeout: 5s
      retries: 5
      start_period: 15s
    restart: on-failure:3
    deploy:
      resources:
        limits:
          memory: 512M
        reservations:
          memory: 256M

  postgresdb:
    image: postgres:alpine
    container_name: "test_postgres"
    environment:
      POSTGRES_PASSWORD: datahub
      # POSTGRES_DB: postgres  # Connectors use the default value: postgres
    volumes:  
      - ./../kafka-connect/setup/mysql-setup.sql:/docker-entrypoint-initdb.d/postgres_setup.sql
    ports:
      - "5432:5432"
    # Debezium doesn't poll tables - it reads the PostgreSQL Write-Ahead Log (WAL) to capture changes in real-time. This is called Change Data Capture (CDC).
    command: >
      postgres
      -c wal_level=logical
      -c max_replication_slots=4
      -c max_wal_senders=4
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 5s
      timeout: 5s
      retries: 5
      start_period: 10s
    restart: on-failure:3
    deploy:
      resources:
        limits:
          memory: 256M
        reservations:
          memory: 128M

  sqlserverdb:
    image: mcr.microsoft.com/mssql/server:2019-latest
    container_name: "test_sqlserver"
    environment:
      ACCEPT_EULA: Y
      SA_PASSWORD: Password123!
      # Developer edition because CDC (Change Data Capture) is required for Debezium
      MSSQL_PID: Developer
      MSSQL_AGENT_ENABLED: true
    volumes:
      - ./../kafka-connect/setup/sqlserver-setup.sql:/usr/src/app/sqlserver-setup.sql
      - ./../kafka-connect/setup/init-sqlserver.sh:/usr/src/app/init-sqlserver.sh
    ports:
      - "1433:1433"
    entrypoint: ["/bin/bash", "/usr/src/app/init-sqlserver.sh"]
    healthcheck:
      test: ["CMD-SHELL", "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'Password123!' -C -d TestDB -Q 'SELECT COUNT(*) FROM dbo.test_table' || exit 1"]
      interval: 15s
      timeout: 10s
      retries: 10
      start_period: 120s
    restart: on-failure:3
    deploy:
      resources:
        limits:
          memory: 2G
        reservations:
          memory: 1G

  mongo:
    hostname: mongo
    image: mongo:6.0.7
    container_name: "test_mongo"
    ports:
      - "27017:27017"
    command: --replSet rs0
    environment:
      # Don't set mongo admin user or password to avoid KeyFile authentication
      # which is required when running MongoDB as a replica set
      - MONGO_INITDB_DATABASE=test_db
    volumes:
      - ./../kafka-connect/setup/conf/:/scripts/
    healthcheck:
      test: ["CMD", "mongosh", "--eval", "db.runCommand({ ping: 1 })"]
      interval: 5s
      timeout: 5s
      retries: 5
      start_period: 15s
    restart: on-failure:3
    deploy:
      resources:
        limits:
          memory: 512M
        reservations:
          memory: 256M

  s3mock:
    image: adobe/s3mock:2.13.0
    environment:
      - initialBuckets=test-bucket
    ports:
      - "9090:9090"
    healthcheck:
      test: ["CMD-SHELL", "curl -s -f http://localhost:9090/ || exit 1"]
      interval: 5s
      timeout: 3s
      retries: 3
      start_period: 10s
    restart: on-failure:3
    deploy:
      resources:
        limits:
          memory: 256M
        reservations:
          memory: 128M