mirror of
https://github.com/datahub-project/datahub.git
synced 2025-07-13 04:02:34 +00:00
ecc01b9a46
* fix(security): commons-text in frontend * refactor(restli): set threads based on cpu cores feat(mce-consumers): hit local restli endpoint * testing docker build * Add retry configuration options for entity client * Kafka debugging * fix(kafka-setup): parallelize topic creation * Adjust docker build * Docker build updates * WIP * fix(lint): metadata-ingestion lint * fix(gradle-docker): fix docker frontend dep * fix(elastic): fix race condition between gms and mae for index creation * Revert "fix(elastic): fix race condition between gms and mae for index creation" This reverts commit 9629d12c3bdb3c0dab87604d409ca4c642c9c6d3. * fix(test): fix datahub frontend test for clean/test cycle * fix(test): datahub-frontend missing assets in test * fix(security): set protobuf lib datahub-upgrade & mce/mae-consumer * gitingore update * fix(docker): remove platform on docker base image, set by buildx * refactor(kafka-producer): update kafka producer tracking/logging * updates per PR feedback * Add documentation around mce standalone consumer Kafka consumer concurrency to follow thread count for restli & sql connection pool Co-authored-by: leifker <dleifker@gmail.com> Co-authored-by: Pedro Silva <pedro@acryl.io>
55 lines
2.2 KiB
YAML
55 lines
2.2 KiB
YAML
services:
|
|
datahub-gms:
|
|
environment:
|
|
- MAE_CONSUMER_ENABLED=false
|
|
- MCE_CONSUMER_ENABLED=false
|
|
datahub-mae-consumer:
|
|
container_name: datahub-mae-consumer
|
|
environment:
|
|
- DATAHUB_GMS_HOST=datahub-gms
|
|
- DATAHUB_GMS_PORT=8080
|
|
- MAE_CONSUMER_ENABLED=true
|
|
- PE_CONSUMER_ENABLED=true
|
|
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
|
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
|
|
- ELASTICSEARCH_HOST=elasticsearch
|
|
- ELASTICSEARCH_PORT=9200
|
|
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
|
|
- GRAPH_SERVICE_IMPL=elasticsearch
|
|
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mae-consumer/resources/entity-registry.yml
|
|
hostname: datahub-mae-consumer
|
|
image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head}
|
|
ports:
|
|
- 9091:9091
|
|
datahub-mce-consumer:
|
|
container_name: datahub-mce-consumer
|
|
environment:
|
|
- MCE_CONSUMER_ENABLED=true
|
|
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
|
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
|
|
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
|
|
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
|
|
- EBEAN_DATASOURCE_USERNAME=datahub
|
|
- EBEAN_DATASOURCE_PASSWORD=datahub
|
|
- EBEAN_DATASOURCE_HOST=mysql:3306
|
|
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
|
|
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
|
|
- ELASTICSEARCH_HOST=elasticsearch
|
|
- ELASTICSEARCH_PORT=9200
|
|
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
|
|
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
|
|
- GRAPH_SERVICE_IMPL=elasticsearch
|
|
- JAVA_OPTS=-Xms1g -Xmx1g
|
|
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-mce-consumer/resources/entity-registry.yml
|
|
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
|
|
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
|
|
- ENTITY_SERVICE_ENABLE_RETENTION=true
|
|
- MAE_CONSUMER_ENABLED=false
|
|
- PE_CONSUMER_ENABLED=false
|
|
- UI_INGESTION_ENABLED=false
|
|
hostname: datahub-mce-consumer
|
|
image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head}
|
|
ports:
|
|
- 9090:9090
|
|
version: '2.3'
|