mirror of
https://github.com/datahub-project/datahub.git
synced 2025-09-15 12:12:07 +00:00
fix(cli): remove duplicate labels from quickstart files (#7886)
This commit is contained in:
Harshal Sheth
GitHub
parent
29e5cfd643
commit
8f8c11081d
@ -48,9 +48,7 @@ services:
|
||||
- DATAHUB_SECRET=YouKnowNothing
|
||||
- DATAHUB_APP_VERSION=1.0
|
||||
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
|
||||
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
|
||||
-Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
|
||||
-Dlogback.debug=false -Dpidfile.path=/dev/null
|
||||
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
|
||||
- ELASTIC_CLIENT_HOST=elasticsearch
|
||||
@ -140,8 +138,7 @@ services:
|
||||
start_period: 2m
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s'
|
||||
|| exit 1
|
||||
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
|
||||
hostname: elasticsearch
|
||||
image: elasticsearch:7.10.1
|
||||
mem_limit: 1g
|
||||
|
||||
@ -46,9 +46,7 @@ services:
|
||||
- DATAHUB_SECRET=YouKnowNothing
|
||||
- DATAHUB_APP_VERSION=1.0
|
||||
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
|
||||
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
|
||||
-Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
|
||||
-Dlogback.debug=false -Dpidfile.path=/dev/null
|
||||
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
|
||||
- ELASTIC_CLIENT_HOST=elasticsearch
|
||||
@ -95,8 +93,6 @@ services:
|
||||
volumes:
|
||||
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
|
||||
datahub-upgrade:
|
||||
labels:
|
||||
datahub_setup_job: true
|
||||
command:
|
||||
- -u
|
||||
- SystemUpdate
|
||||
@ -133,8 +129,7 @@ services:
|
||||
start_period: 2m
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s'
|
||||
|| exit 1
|
||||
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
|
||||
hostname: elasticsearch
|
||||
image: elasticsearch:7.10.1
|
||||
mem_limit: 1g
|
||||
@ -143,8 +138,6 @@ services:
|
||||
volumes:
|
||||
- esdata:/usr/share/elasticsearch/data
|
||||
elasticsearch-setup:
|
||||
labels:
|
||||
datahub_setup_job: true
|
||||
container_name: elasticsearch-setup
|
||||
depends_on:
|
||||
- elasticsearch
|
||||
@ -157,8 +150,6 @@ services:
|
||||
labels:
|
||||
datahub_setup_job: true
|
||||
kafka-setup:
|
||||
labels:
|
||||
datahub_setup_job: true
|
||||
container_name: kafka-setup
|
||||
depends_on:
|
||||
- broker
|
||||
@ -187,8 +178,6 @@ services:
|
||||
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
|
||||
- mysqldata:/var/lib/mysql
|
||||
mysql-setup:
|
||||
labels:
|
||||
datahub_setup_job: true
|
||||
container_name: mysql-setup
|
||||
depends_on:
|
||||
- mysql
|
||||
|
||||
@ -5,115 +5,115 @@ services:
|
||||
broker:
|
||||
container_name: broker
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- zookeeper
|
||||
environment:
|
||||
- KAFKA_BROKER_ID=1
|
||||
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
|
||||
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
|
||||
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
|
||||
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
|
||||
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
|
||||
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
|
||||
- KAFKA_BROKER_ID=1
|
||||
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
|
||||
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
|
||||
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
|
||||
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
|
||||
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
|
||||
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
|
||||
hostname: broker
|
||||
image: confluentinc/cp-kafka:7.2.2
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
|
||||
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
|
||||
datahub-actions:
|
||||
depends_on:
|
||||
- datahub-gms
|
||||
- datahub-gms
|
||||
environment:
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- DATAHUB_GMS_PROTOCOL=http
|
||||
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
|
||||
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
|
||||
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
|
||||
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
|
||||
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- DATAHUB_GMS_PROTOCOL=http
|
||||
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
|
||||
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
|
||||
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
|
||||
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
|
||||
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
|
||||
hostname: actions
|
||||
image: acryldata/datahub-actions:${ACTIONS_VERSION:-head}
|
||||
restart: on-failure:5
|
||||
datahub-frontend-react:
|
||||
container_name: datahub-frontend-react
|
||||
depends_on:
|
||||
- datahub-gms
|
||||
- datahub-gms
|
||||
environment:
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- DATAHUB_SECRET=YouKnowNothing
|
||||
- DATAHUB_APP_VERSION=1.0
|
||||
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
|
||||
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
|
||||
- ELASTIC_CLIENT_HOST=elasticsearch
|
||||
- ELASTIC_CLIENT_PORT=9200
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- DATAHUB_SECRET=YouKnowNothing
|
||||
- DATAHUB_APP_VERSION=1.0
|
||||
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
|
||||
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
|
||||
- ELASTIC_CLIENT_HOST=elasticsearch
|
||||
- ELASTIC_CLIENT_PORT=9200
|
||||
hostname: datahub-frontend-react
|
||||
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
|
||||
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
|
||||
volumes:
|
||||
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
|
||||
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
|
||||
datahub-gms:
|
||||
container_name: datahub-gms
|
||||
depends_on:
|
||||
- mysql
|
||||
- mysql
|
||||
environment:
|
||||
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
|
||||
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
|
||||
- DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
|
||||
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
|
||||
- EBEAN_DATASOURCE_HOST=mysql:3306
|
||||
- EBEAN_DATASOURCE_PASSWORD=datahub
|
||||
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
|
||||
- EBEAN_DATASOURCE_USERNAME=datahub
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
|
||||
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
|
||||
- ENTITY_SERVICE_ENABLE_RETENTION=true
|
||||
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
|
||||
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
|
||||
- GRAPH_SERVICE_IMPL=elasticsearch
|
||||
- JAVA_OPTS=-Xms1g -Xmx1g
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
|
||||
- MAE_CONSUMER_ENABLED=true
|
||||
- MCE_CONSUMER_ENABLED=true
|
||||
- PE_CONSUMER_ENABLED=true
|
||||
- UI_INGESTION_ENABLED=true
|
||||
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
|
||||
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
|
||||
- DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
|
||||
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
|
||||
- EBEAN_DATASOURCE_HOST=mysql:3306
|
||||
- EBEAN_DATASOURCE_PASSWORD=datahub
|
||||
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
|
||||
- EBEAN_DATASOURCE_USERNAME=datahub
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
|
||||
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
|
||||
- ENTITY_SERVICE_ENABLE_RETENTION=true
|
||||
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
|
||||
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
|
||||
- GRAPH_SERVICE_IMPL=elasticsearch
|
||||
- JAVA_OPTS=-Xms1g -Xmx1g
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
|
||||
- MAE_CONSUMER_ENABLED=true
|
||||
- MCE_CONSUMER_ENABLED=true
|
||||
- PE_CONSUMER_ENABLED=true
|
||||
- UI_INGESTION_ENABLED=true
|
||||
hostname: datahub-gms
|
||||
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
|
||||
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
|
||||
volumes:
|
||||
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
|
||||
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
|
||||
datahub-upgrade:
|
||||
command:
|
||||
- -u
|
||||
- SystemUpdate
|
||||
- -u
|
||||
- SystemUpdate
|
||||
container_name: datahub-upgrade
|
||||
environment:
|
||||
- EBEAN_DATASOURCE_USERNAME=datahub
|
||||
- EBEAN_DATASOURCE_PASSWORD=datahub
|
||||
- EBEAN_DATASOURCE_HOST=mysql:3306
|
||||
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
|
||||
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
|
||||
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
|
||||
- ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
|
||||
- GRAPH_SERVICE_IMPL=elasticsearch
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
|
||||
- EBEAN_DATASOURCE_USERNAME=datahub
|
||||
- EBEAN_DATASOURCE_PASSWORD=datahub
|
||||
- EBEAN_DATASOURCE_HOST=mysql:3306
|
||||
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
|
||||
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
|
||||
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
|
||||
- ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
|
||||
- GRAPH_SERVICE_IMPL=elasticsearch
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
|
||||
hostname: datahub-upgrade
|
||||
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
|
||||
labels:
|
||||
@ -121,30 +121,30 @@ services:
|
||||
elasticsearch:
|
||||
container_name: elasticsearch
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- xpack.security.enabled=false
|
||||
- ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true
|
||||
- discovery.type=single-node
|
||||
- xpack.security.enabled=false
|
||||
- ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true
|
||||
healthcheck:
|
||||
retries: 4
|
||||
start_period: 2m
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
|
||||
- CMD-SHELL
|
||||
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
|
||||
hostname: elasticsearch
|
||||
image: elasticsearch:7.10.1
|
||||
mem_limit: 1g
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
|
||||
- ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
|
||||
volumes:
|
||||
- esdata:/usr/share/elasticsearch/data
|
||||
- esdata:/usr/share/elasticsearch/data
|
||||
elasticsearch-setup:
|
||||
container_name: elasticsearch-setup
|
||||
depends_on:
|
||||
- elasticsearch
|
||||
- elasticsearch
|
||||
environment:
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ELASTICSEARCH_PROTOCOL=http
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ELASTICSEARCH_PROTOCOL=http
|
||||
hostname: elasticsearch-setup
|
||||
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
|
||||
labels:
|
||||
@ -152,12 +152,12 @@ services:
|
||||
kafka-setup:
|
||||
container_name: kafka-setup
|
||||
depends_on:
|
||||
- broker
|
||||
- schema-registry
|
||||
- broker
|
||||
- schema-registry
|
||||
environment:
|
||||
- DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false}
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
|
||||
- DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false}
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
|
||||
hostname: kafka-setup
|
||||
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
|
||||
labels:
|
||||
@ -166,55 +166,55 @@ services:
|
||||
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
|
||||
container_name: mysql
|
||||
environment:
|
||||
- MYSQL_DATABASE=datahub
|
||||
- MYSQL_USER=datahub
|
||||
- MYSQL_PASSWORD=datahub
|
||||
- MYSQL_ROOT_PASSWORD=datahub
|
||||
- MYSQL_DATABASE=datahub
|
||||
- MYSQL_USER=datahub
|
||||
- MYSQL_PASSWORD=datahub
|
||||
- MYSQL_ROOT_PASSWORD=datahub
|
||||
hostname: mysql
|
||||
image: mysql:5.7
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
|
||||
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
|
||||
volumes:
|
||||
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
|
||||
- mysqldata:/var/lib/mysql
|
||||
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
|
||||
- mysqldata:/var/lib/mysql
|
||||
mysql-setup:
|
||||
labels:
|
||||
datahub_setup_job: true
|
||||
container_name: mysql-setup
|
||||
depends_on:
|
||||
- mysql
|
||||
- mysql
|
||||
environment:
|
||||
- MYSQL_HOST=mysql
|
||||
- MYSQL_PORT=3306
|
||||
- MYSQL_USERNAME=datahub
|
||||
- MYSQL_PASSWORD=datahub
|
||||
- DATAHUB_DB_NAME=datahub
|
||||
- MYSQL_HOST=mysql
|
||||
- MYSQL_PORT=3306
|
||||
- MYSQL_USERNAME=datahub
|
||||
- MYSQL_PASSWORD=datahub
|
||||
- DATAHUB_DB_NAME=datahub
|
||||
hostname: mysql-setup
|
||||
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head}
|
||||
labels:
|
||||
datahub_setup_job: true
|
||||
schema-registry:
|
||||
container_name: schema-registry
|
||||
depends_on:
|
||||
- broker
|
||||
- broker
|
||||
environment:
|
||||
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
|
||||
- SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
|
||||
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
|
||||
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
|
||||
- SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
|
||||
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
|
||||
hostname: schema-registry
|
||||
image: confluentinc/cp-schema-registry:7.2.2
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
|
||||
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
|
||||
zookeeper:
|
||||
container_name: zookeeper
|
||||
environment:
|
||||
- ZOOKEEPER_CLIENT_PORT=2181
|
||||
- ZOOKEEPER_TICK_TIME=2000
|
||||
- ZOOKEEPER_CLIENT_PORT=2181
|
||||
- ZOOKEEPER_TICK_TIME=2000
|
||||
hostname: zookeeper
|
||||
image: confluentinc/cp-zookeeper:7.2.2
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
|
||||
- ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
|
||||
volumes:
|
||||
- zkdata:/var/lib/zookeeper
|
||||
version: "2.3"
|
||||
- zkdata:/var/lib/zookeeper
|
||||
version: '2.3'
|
||||
volumes:
|
||||
esdata: null
|
||||
mysqldata: null
|
||||
|
||||
@ -1,47 +1,47 @@
|
||||
services:
|
||||
datahub-frontend-react:
|
||||
environment:
|
||||
- ENABLE_PROMETHEUS=true
|
||||
- ENABLE_OTEL=true
|
||||
- OTEL_TRACES_EXPORTER=jaeger
|
||||
- OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
|
||||
- OTEL_METRICS_EXPORTER=none
|
||||
- OTEL_SERVICE_NAME=datahub-gms
|
||||
- ENABLE_PROMETHEUS=true
|
||||
- ENABLE_OTEL=true
|
||||
- OTEL_TRACES_EXPORTER=jaeger
|
||||
- OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
|
||||
- OTEL_METRICS_EXPORTER=none
|
||||
- OTEL_SERVICE_NAME=datahub-gms
|
||||
ports:
|
||||
- "4318"
|
||||
- '4318'
|
||||
datahub-gms:
|
||||
environment:
|
||||
- ENABLE_PROMETHEUS=true
|
||||
- ENABLE_OTEL=true
|
||||
- OTEL_TRACES_EXPORTER=jaeger
|
||||
- OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
|
||||
- OTEL_METRICS_EXPORTER=none
|
||||
- OTEL_SERVICE_NAME=datahub-gms
|
||||
- ENABLE_PROMETHEUS=true
|
||||
- ENABLE_OTEL=true
|
||||
- OTEL_TRACES_EXPORTER=jaeger
|
||||
- OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
|
||||
- OTEL_METRICS_EXPORTER=none
|
||||
- OTEL_SERVICE_NAME=datahub-gms
|
||||
ports:
|
||||
- "4318"
|
||||
- '4318'
|
||||
grafana:
|
||||
depends_on:
|
||||
- prometheus
|
||||
- prometheus
|
||||
image: grafana/grafana:9.1.4
|
||||
ports:
|
||||
- 3001:3000
|
||||
- 3001:3000
|
||||
volumes:
|
||||
- grafana-storage:/var/lib/grafana
|
||||
- ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources
|
||||
- ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards
|
||||
- grafana-storage:/var/lib/grafana
|
||||
- ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources
|
||||
- ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards
|
||||
jaeger-all-in-one:
|
||||
image: jaegertracing/all-in-one:latest
|
||||
ports:
|
||||
- 16686:16686
|
||||
- "14268"
|
||||
- "14250"
|
||||
- 16686:16686
|
||||
- '14268'
|
||||
- '14250'
|
||||
prometheus:
|
||||
container_name: prometheus
|
||||
image: prom/prometheus:latest
|
||||
ports:
|
||||
- 9089:9090
|
||||
- 9089:9090
|
||||
volumes:
|
||||
- ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml
|
||||
version: "2.3"
|
||||
- ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml
|
||||
version: '2.3'
|
||||
volumes:
|
||||
grafana-storage: null
|
||||
|
||||
@ -5,124 +5,124 @@ services:
|
||||
broker:
|
||||
container_name: broker
|
||||
depends_on:
|
||||
- zookeeper
|
||||
- zookeeper
|
||||
environment:
|
||||
- KAFKA_BROKER_ID=1
|
||||
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
|
||||
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
|
||||
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
|
||||
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
|
||||
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
|
||||
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
|
||||
- KAFKA_BROKER_ID=1
|
||||
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
|
||||
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
||||
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
|
||||
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
|
||||
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
|
||||
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
|
||||
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
|
||||
hostname: broker
|
||||
image: confluentinc/cp-kafka:7.2.2
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
|
||||
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
|
||||
volumes:
|
||||
- broker:/var/lib/kafka/data/
|
||||
- broker:/var/lib/kafka/data/
|
||||
datahub-actions:
|
||||
depends_on:
|
||||
- datahub-gms
|
||||
- datahub-gms
|
||||
environment:
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- DATAHUB_GMS_PROTOCOL=http
|
||||
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
|
||||
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
|
||||
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
|
||||
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
|
||||
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- DATAHUB_GMS_PROTOCOL=http
|
||||
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
|
||||
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
|
||||
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
|
||||
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
|
||||
- SCHEMA_REGISTRY_URL=http://schema-registry:8081
|
||||
hostname: actions
|
||||
image: acryldata/datahub-actions:${ACTIONS_VERSION:-head}
|
||||
restart: on-failure:5
|
||||
datahub-frontend-react:
|
||||
container_name: datahub-frontend-react
|
||||
depends_on:
|
||||
- datahub-gms
|
||||
- datahub-gms
|
||||
environment:
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- DATAHUB_SECRET=YouKnowNothing
|
||||
- DATAHUB_APP_VERSION=1.0
|
||||
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
|
||||
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
|
||||
- ELASTIC_CLIENT_HOST=elasticsearch
|
||||
- ELASTIC_CLIENT_PORT=9200
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- DATAHUB_SECRET=YouKnowNothing
|
||||
- DATAHUB_APP_VERSION=1.0
|
||||
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
|
||||
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
|
||||
- ELASTIC_CLIENT_HOST=elasticsearch
|
||||
- ELASTIC_CLIENT_PORT=9200
|
||||
hostname: datahub-frontend-react
|
||||
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
|
||||
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
|
||||
volumes:
|
||||
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
|
||||
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
|
||||
datahub-gms:
|
||||
container_name: datahub-gms
|
||||
depends_on:
|
||||
- mysql
|
||||
- neo4j
|
||||
- mysql
|
||||
- neo4j
|
||||
environment:
|
||||
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
|
||||
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
|
||||
- DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
|
||||
- EBEAN_DATASOURCE_USERNAME=datahub
|
||||
- EBEAN_DATASOURCE_PASSWORD=datahub
|
||||
- EBEAN_DATASOURCE_HOST=mysql:3306
|
||||
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
|
||||
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
|
||||
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
|
||||
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
|
||||
- NEO4J_HOST=http://neo4j:7474
|
||||
- NEO4J_URI=bolt://neo4j
|
||||
- NEO4J_USERNAME=neo4j
|
||||
- NEO4J_PASSWORD=datahub
|
||||
- JAVA_OPTS=-Xms1g -Xmx1g
|
||||
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
|
||||
- GRAPH_SERVICE_IMPL=neo4j
|
||||
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
|
||||
- ENTITY_SERVICE_ENABLE_RETENTION=true
|
||||
- MAE_CONSUMER_ENABLED=true
|
||||
- MCE_CONSUMER_ENABLED=true
|
||||
- PE_CONSUMER_ENABLED=true
|
||||
- UI_INGESTION_ENABLED=true
|
||||
- METADATA_SERVICE_AUTH_ENABLED=false
|
||||
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
|
||||
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
|
||||
- DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
|
||||
- EBEAN_DATASOURCE_USERNAME=datahub
|
||||
- EBEAN_DATASOURCE_PASSWORD=datahub
|
||||
- EBEAN_DATASOURCE_HOST=mysql:3306
|
||||
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
|
||||
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
|
||||
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
|
||||
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
|
||||
- NEO4J_HOST=http://neo4j:7474
|
||||
- NEO4J_URI=bolt://neo4j
|
||||
- NEO4J_USERNAME=neo4j
|
||||
- NEO4J_PASSWORD=datahub
|
||||
- JAVA_OPTS=-Xms1g -Xmx1g
|
||||
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
|
||||
- GRAPH_SERVICE_IMPL=neo4j
|
||||
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
|
||||
- ENTITY_SERVICE_ENABLE_RETENTION=true
|
||||
- MAE_CONSUMER_ENABLED=true
|
||||
- MCE_CONSUMER_ENABLED=true
|
||||
- PE_CONSUMER_ENABLED=true
|
||||
- UI_INGESTION_ENABLED=true
|
||||
- METADATA_SERVICE_AUTH_ENABLED=false
|
||||
hostname: datahub-gms
|
||||
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
|
||||
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
|
||||
volumes:
|
||||
- ${HOME}/.datahub/plugins/:/etc/datahub/plugins
|
||||
- ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources
|
||||
- ${HOME}/.datahub/plugins/:/etc/datahub/plugins
|
||||
- ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources
|
||||
datahub-upgrade:
|
||||
command:
|
||||
- -u
|
||||
- SystemUpdate
|
||||
- -u
|
||||
- SystemUpdate
|
||||
container_name: datahub-upgrade
|
||||
environment:
|
||||
- EBEAN_DATASOURCE_USERNAME=datahub
|
||||
- EBEAN_DATASOURCE_PASSWORD=datahub
|
||||
- EBEAN_DATASOURCE_HOST=mysql:3306
|
||||
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
|
||||
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
|
||||
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
|
||||
- ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
|
||||
- GRAPH_SERVICE_IMPL=elasticsearch
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
|
||||
- EBEAN_DATASOURCE_USERNAME=datahub
|
||||
- EBEAN_DATASOURCE_PASSWORD=datahub
|
||||
- EBEAN_DATASOURCE_HOST=mysql:3306
|
||||
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
|
||||
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
|
||||
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
|
||||
- ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
|
||||
- GRAPH_SERVICE_IMPL=elasticsearch
|
||||
- DATAHUB_GMS_HOST=datahub-gms
|
||||
- DATAHUB_GMS_PORT=8080
|
||||
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
|
||||
hostname: datahub-upgrade
|
||||
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
|
||||
labels:
|
||||
@ -130,30 +130,30 @@ services:
|
||||
elasticsearch:
|
||||
container_name: elasticsearch
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- xpack.security.enabled=false
|
||||
- ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true
|
||||
- discovery.type=single-node
|
||||
- xpack.security.enabled=false
|
||||
- ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true
|
||||
healthcheck:
|
||||
retries: 4
|
||||
start_period: 2m
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
|
||||
- CMD-SHELL
|
||||
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
|
||||
hostname: elasticsearch
|
||||
image: elasticsearch:7.10.1
|
||||
mem_limit: 1g
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
|
||||
- ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
|
||||
volumes:
|
||||
- esdata:/usr/share/elasticsearch/data
|
||||
- esdata:/usr/share/elasticsearch/data
|
||||
elasticsearch-setup:
|
||||
container_name: elasticsearch-setup
|
||||
depends_on:
|
||||
- elasticsearch
|
||||
- elasticsearch
|
||||
environment:
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ELASTICSEARCH_PROTOCOL=http
|
||||
- ELASTICSEARCH_HOST=elasticsearch
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- ELASTICSEARCH_PROTOCOL=http
|
||||
hostname: elasticsearch-setup
|
||||
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
|
||||
labels:
|
||||
@ -161,12 +161,12 @@ services:
|
||||
kafka-setup:
|
||||
container_name: kafka-setup
|
||||
depends_on:
|
||||
- broker
|
||||
- schema-registry
|
||||
- broker
|
||||
- schema-registry
|
||||
environment:
|
||||
- DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false}
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
|
||||
- DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false}
|
||||
- KAFKA_BOOTSTRAP_SERVER=broker:29092
|
||||
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
|
||||
hostname: kafka-setup
|
||||
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
|
||||
labels:
|
||||
@ -175,27 +175,27 @@ services:
|
||||
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
|
||||
container_name: mysql
|
||||
environment:
|
||||
- MYSQL_DATABASE=datahub
|
||||
- MYSQL_USER=datahub
|
||||
- MYSQL_PASSWORD=datahub
|
||||
- MYSQL_ROOT_PASSWORD=datahub
|
||||
- MYSQL_DATABASE=datahub
|
||||
- MYSQL_USER=datahub
|
||||
- MYSQL_PASSWORD=datahub
|
||||
- MYSQL_ROOT_PASSWORD=datahub
|
||||
hostname: mysql
|
||||
image: mysql:5.7
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
|
||||
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
|
||||
volumes:
|
||||
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
|
||||
- mysqldata:/var/lib/mysql
|
||||
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
|
||||
- mysqldata:/var/lib/mysql
|
||||
mysql-setup:
|
||||
container_name: mysql-setup
|
||||
depends_on:
|
||||
- mysql
|
||||
- mysql
|
||||
environment:
|
||||
- MYSQL_HOST=mysql
|
||||
- MYSQL_PORT=3306
|
||||
- MYSQL_USERNAME=datahub
|
||||
- MYSQL_PASSWORD=datahub
|
||||
- DATAHUB_DB_NAME=datahub
|
||||
- MYSQL_HOST=mysql
|
||||
- MYSQL_PORT=3306
|
||||
- MYSQL_USERNAME=datahub
|
||||
- MYSQL_PASSWORD=datahub
|
||||
- DATAHUB_DB_NAME=datahub
|
||||
hostname: mysql-setup
|
||||
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head}
|
||||
labels:
|
||||
@ -203,40 +203,40 @@ services:
|
||||
neo4j:
|
||||
container_name: neo4j
|
||||
environment:
|
||||
- NEO4J_AUTH=neo4j/datahub
|
||||
- NEO4J_dbms_default__database=graph.db
|
||||
- NEO4J_dbms_allow__upgrade=true
|
||||
- NEO4J_AUTH=neo4j/datahub
|
||||
- NEO4J_dbms_default__database=graph.db
|
||||
- NEO4J_dbms_allow__upgrade=true
|
||||
hostname: neo4j
|
||||
image: neo4j:4.4.9-community
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474
|
||||
- ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687
|
||||
- ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474
|
||||
- ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687
|
||||
volumes:
|
||||
- neo4jdata:/data
|
||||
- neo4jdata:/data
|
||||
schema-registry:
|
||||
container_name: schema-registry
|
||||
depends_on:
|
||||
- broker
|
||||
- broker
|
||||
environment:
|
||||
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
|
||||
- SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
|
||||
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
|
||||
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
|
||||
- SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
|
||||
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
|
||||
hostname: schema-registry
|
||||
image: confluentinc/cp-schema-registry:7.2.2
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
|
||||
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
|
||||
zookeeper:
|
||||
container_name: zookeeper
|
||||
environment:
|
||||
- ZOOKEEPER_CLIENT_PORT=2181
|
||||
- ZOOKEEPER_TICK_TIME=2000
|
||||
- ZOOKEEPER_CLIENT_PORT=2181
|
||||
- ZOOKEEPER_TICK_TIME=2000
|
||||
hostname: zookeeper
|
||||
image: confluentinc/cp-zookeeper:7.2.2
|
||||
ports:
|
||||
- ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
|
||||
- ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
|
||||
volumes:
|
||||
- zkdata:/var/lib/zookeeper
|
||||
version: "2.3"
|
||||
- zkdata:/var/lib/zookeeper
|
||||
version: '2.3'
|
||||
volumes:
|
||||
broker: null
|
||||
esdata: null
|
||||
|
||||
@ -1,8 +1,5 @@
|
||||
#!/bin/bash
|
||||
|
||||
# this scripts checks if docker-compose$flavour.quickstart.yml is up to date for these 'flavours':
|
||||
FLAVOURS=("" "-without-neo4j" ".monitoring")
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$DIR"
|
||||
|
||||
@ -12,21 +9,4 @@ python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
|
||||
pip install -r requirements.txt
|
||||
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml temp.quickstart.yml
|
||||
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml temp-without-neo4j.quickstart.yml
|
||||
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose.m1.yml temp-m1.quickstart.yml
|
||||
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml ../docker-compose-without-neo4j.m1.yml temp-without-neo4j-m1.quickstart.yml
|
||||
python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.monitoring.quickstart.yml
|
||||
python generate_docker_quickstart.py ../docker-compose.consumers.yml temp.consumers.quickstart.yml
|
||||
python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml temp.consumers-without-neo4j.quickstart.yml
|
||||
|
||||
for flavour in "${FLAVOURS[@]}"
|
||||
do
|
||||
|
||||
if cmp <(yq -i -P 'sort_keys(..)' docker-compose$flavour.quickstart.yml) <(yq -i -P 'sort_keys(..)' temp$flavour.quickstart.yml); then
|
||||
echo "docker-compose$flavour.quickstart.yml is up to date."
|
||||
else
|
||||
echo "docker-compose$flavour.quickstart.yml is out of date."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
python generate_docker_quickstart.py check-all
|
||||
|
||||
@ -1,4 +1,8 @@
|
||||
from io import StringIO
|
||||
from typing import List
|
||||
import os
|
||||
import sys
|
||||
import pathlib
|
||||
from collections.abc import Mapping
|
||||
|
||||
import click
|
||||
@ -6,8 +10,38 @@ import yaml
|
||||
from dotenv import dotenv_values
|
||||
from yaml import Loader
|
||||
|
||||
# Generates a merged docker-compose file with env variables inlined.
|
||||
# Usage: python3 docker_compose_cli_gen.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose-gen.yml
|
||||
COMPOSE_SPECS = {
|
||||
"docker-compose.quickstart.yml": [
|
||||
"../docker-compose.yml",
|
||||
"../docker-compose.override.yml",
|
||||
],
|
||||
"docker-compose-m1.quickstart.yml": [
|
||||
"../docker-compose.yml",
|
||||
"../docker-compose.override.yml",
|
||||
"../docker-compose.m1.yml",
|
||||
],
|
||||
"docker-compose-without-neo4j.quickstart.yml": [
|
||||
"../docker-compose-without-neo4j.yml",
|
||||
"../docker-compose-without-neo4j.override.yml",
|
||||
],
|
||||
"docker-compose-without-neo4j-m1.quickstart.yml": [
|
||||
"../docker-compose-without-neo4j.yml",
|
||||
"../docker-compose-without-neo4j.override.yml",
|
||||
"../docker-compose-without-neo4j.m1.yml",
|
||||
],
|
||||
"docker-compose.monitoring.quickstart.yml": [
|
||||
"../monitoring/docker-compose.monitoring.yml",
|
||||
],
|
||||
"docker-compose.consumers.quickstart.yml": [
|
||||
"../docker-compose.consumers.yml",
|
||||
],
|
||||
"docker-compose.consumers-without-neo4j.quickstart.yml": [
|
||||
"../docker-compose.consumers-without-neo4j.yml",
|
||||
],
|
||||
"docker-compose.kafka-setup.quickstart.yml": [
|
||||
"../docker-compose.kafka-setup.yml",
|
||||
],
|
||||
}
|
||||
|
||||
omitted_services = [
|
||||
"kafka-rest-proxy",
|
||||
@ -34,6 +68,7 @@ def dict_merge(dct, merge_dct):
|
||||
else:
|
||||
dct[k] = merge_dct[k]
|
||||
|
||||
|
||||
def modify_docker_config(base_path, docker_yaml_config):
|
||||
if not docker_yaml_config["services"]:
|
||||
docker_yaml_config["services"] = {}
|
||||
@ -91,17 +126,35 @@ def modify_docker_config(base_path, docker_yaml_config):
|
||||
docker_yaml_config["version"] = "2.3"
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument(
|
||||
"compose-files",
|
||||
nargs=-1,
|
||||
type=click.Path(
|
||||
exists=True,
|
||||
dir_okay=False,
|
||||
),
|
||||
)
|
||||
@click.argument("output-file", type=click.Path())
|
||||
def generate(compose_files, output_file) -> None:
|
||||
def dedup_env_vars(merged_docker_config):
|
||||
for service in merged_docker_config["services"]:
|
||||
if "environment" in merged_docker_config["services"][service]:
|
||||
lst = merged_docker_config["services"][service]["environment"]
|
||||
if lst is not None:
|
||||
# use a set to cache duplicates
|
||||
caches = set()
|
||||
results = {}
|
||||
for item in lst:
|
||||
partitions = item.rpartition("=")
|
||||
prefix = partitions[0]
|
||||
suffix = partitions[1]
|
||||
# check whether prefix already exists
|
||||
if prefix not in caches and suffix != "":
|
||||
results[prefix] = item
|
||||
caches.add(prefix)
|
||||
if set(lst) != set([v for k, v in results.items()]):
|
||||
sorted_vars = sorted([k for k in results])
|
||||
merged_docker_config["services"][service]["environment"] = [
|
||||
results[var] for var in sorted_vars
|
||||
]
|
||||
|
||||
|
||||
def merge_files(compose_files: List[str]) -> str:
|
||||
"""
|
||||
Generates a merged docker-compose file with env variables inlined.
|
||||
|
||||
Example Usage: python3 generate_docker_quickstart.py generate-one ../docker-compose.yml ../docker-compose.override.yml ../docker-compose-gen.yml
|
||||
"""
|
||||
|
||||
# Resolve .env files to inlined vars
|
||||
modified_files = []
|
||||
@ -121,40 +174,77 @@ def generate(compose_files, output_file) -> None:
|
||||
# Dedup env vars, last wins
|
||||
dedup_env_vars(merged_docker_config)
|
||||
|
||||
# Generate yaml to string.
|
||||
out = StringIO()
|
||||
yaml.dump(
|
||||
merged_docker_config,
|
||||
out,
|
||||
default_flow_style=False,
|
||||
width=1000,
|
||||
)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
@click.group()
|
||||
def main_cmd() -> None:
|
||||
pass
|
||||
|
||||
|
||||
@main_cmd.command()
|
||||
@click.argument(
|
||||
"compose-files",
|
||||
nargs=-1,
|
||||
type=click.Path(
|
||||
exists=True,
|
||||
dir_okay=False,
|
||||
),
|
||||
)
|
||||
@click.argument("output-file", type=click.Path())
|
||||
def generate_one(compose_files, output_file) -> None:
|
||||
"""
|
||||
Generates a merged docker-compose file with env variables inlined.
|
||||
|
||||
Example Usage: python3 generate_docker_quickstart.py generate-one ../docker-compose.yml ../docker-compose.override.yml ../docker-compose-gen.yml
|
||||
"""
|
||||
|
||||
merged_contents = merge_files(compose_files)
|
||||
|
||||
# Write output file
|
||||
output_dir = os.path.dirname(output_file)
|
||||
if len(output_dir) and not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
with open(output_file, "w") as new_conf_file:
|
||||
yaml.dump(
|
||||
merged_docker_config,
|
||||
new_conf_file,
|
||||
default_flow_style=False,
|
||||
)
|
||||
pathlib.Path(output_file).parent.mkdir(parents=True, exist_ok=True)
|
||||
pathlib.Path(output_file).write_text(merged_contents)
|
||||
|
||||
print(f"Successfully generated {output_file}.")
|
||||
|
||||
|
||||
def dedup_env_vars(merged_docker_config):
|
||||
for service in merged_docker_config['services']:
|
||||
if 'environment' in merged_docker_config['services'][service]:
|
||||
lst = merged_docker_config['services'][service]['environment']
|
||||
if lst is not None:
|
||||
# use a set to cache duplicates
|
||||
caches = set()
|
||||
results = {}
|
||||
for item in lst:
|
||||
partitions = item.rpartition('=')
|
||||
prefix = partitions[0]
|
||||
suffix = partitions[1]
|
||||
# check whether prefix already exists
|
||||
if prefix not in caches and suffix != "":
|
||||
results[prefix] = item
|
||||
caches.add(prefix)
|
||||
if set(lst) != set([v for k,v in results.items()]):
|
||||
sorted_vars = sorted([k for k in results])
|
||||
merged_docker_config['services'][service]['environment'] = [results[var] for var in sorted_vars]
|
||||
@main_cmd.command()
|
||||
@click.pass_context
|
||||
def generate_all(ctx: click.Context) -> None:
|
||||
"""
|
||||
Generates all merged docker-compose files with env variables inlined.
|
||||
"""
|
||||
|
||||
for output_compose_file, inputs in COMPOSE_SPECS.items():
|
||||
ctx.invoke(generate_one, compose_files=inputs, output_file=output_compose_file)
|
||||
|
||||
|
||||
@main_cmd.command()
|
||||
def check_all() -> None:
|
||||
"""
|
||||
Checks that the generated docker-compose files are up to date.
|
||||
"""
|
||||
|
||||
for output_compose_file, inputs in COMPOSE_SPECS.items():
|
||||
expected = merge_files(inputs)
|
||||
|
||||
# Check that the files match.
|
||||
current = pathlib.Path(output_compose_file).read_text()
|
||||
|
||||
if expected != current:
|
||||
print(
|
||||
f"File {output_compose_file} is out of date. Please run `python3 generate_docker_quickstart.py generate-all`."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate()
|
||||
main_cmd()
|
||||
|
||||
@ -9,11 +9,4 @@ python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
|
||||
pip install -r requirements.txt
|
||||
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml docker-compose.quickstart.yml
|
||||
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml docker-compose-without-neo4j.quickstart.yml
|
||||
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose.m1.yml docker-compose-m1.quickstart.yml
|
||||
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml ../docker-compose-without-neo4j.m1.yml docker-compose-without-neo4j-m1.quickstart.yml
|
||||
python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.monitoring.quickstart.yml
|
||||
python generate_docker_quickstart.py ../docker-compose.consumers.yml docker-compose.consumers.quickstart.yml
|
||||
python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml docker-compose.consumers-without-neo4j.quickstart.yml
|
||||
python generate_docker_quickstart.py ../docker-compose.kafka-setup.yml docker-compose.kafka-setup.quickstart.yml
|
||||
python generate_docker_quickstart.py generate-all
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user