yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-16 20:53:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(cli): remove duplicate labels from quickstart files (#7886)

Browse Source
This commit is contained in:
Harshal Sheth 2023-04-26 03:05:34 +05:30 committed by GitHub
parent 29e5cfd643
commit 8f8c11081d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 415 additions and 366 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docker/quickstart/docker-compose-m1.quickstart.yml
View File

@ -48,9 +48,7 @@ services:
- DATAHUB_SECRET=YouKnowNothing - DATAHUB_SECRET=YouKnowNothing
- DATAHUB_APP_VERSION=1.0 - DATAHUB_APP_VERSION=1.0
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
-Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
-Dlogback.debug=false -Dpidfile.path=/dev/null
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
- ELASTIC_CLIENT_HOST=elasticsearch - ELASTIC_CLIENT_HOST=elasticsearch
@ -140,8 +138,7 @@ services:
start_period: 2m start_period: 2m
test: test:
- CMD-SHELL - CMD-SHELL
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
|| exit 1
hostname: elasticsearch hostname: elasticsearch
image: elasticsearch:7.10.1 image: elasticsearch:7.10.1
mem_limit: 1g mem_limit: 1g

15
docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
View File

@ -46,9 +46,7 @@ services:
- DATAHUB_SECRET=YouKnowNothing - DATAHUB_SECRET=YouKnowNothing
- DATAHUB_APP_VERSION=1.0 - DATAHUB_APP_VERSION=1.0
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
-Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
-Dlogback.debug=false -Dpidfile.path=/dev/null
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
- ELASTIC_CLIENT_HOST=elasticsearch - ELASTIC_CLIENT_HOST=elasticsearch
@ -95,8 +93,6 @@ services:
volumes: volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins - ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-upgrade: datahub-upgrade:
labels:
datahub_setup_job: true
command: command:
- -u - -u
- SystemUpdate - SystemUpdate
@ -133,8 +129,7 @@ services:
start_period: 2m start_period: 2m
test: test:
- CMD-SHELL - CMD-SHELL
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
|| exit 1
hostname: elasticsearch hostname: elasticsearch
image: elasticsearch:7.10.1 image: elasticsearch:7.10.1
mem_limit: 1g mem_limit: 1g
@ -143,8 +138,6 @@ services:
volumes: volumes:
- esdata:/usr/share/elasticsearch/data - esdata:/usr/share/elasticsearch/data
elasticsearch-setup: elasticsearch-setup:
labels:
datahub_setup_job: true
container_name: elasticsearch-setup container_name: elasticsearch-setup
depends_on: depends_on:
- elasticsearch - elasticsearch
@ -157,8 +150,6 @@ services:
labels: labels:
datahub_setup_job: true datahub_setup_job: true
kafka-setup: kafka-setup:
labels:
datahub_setup_job: true
container_name: kafka-setup container_name: kafka-setup
depends_on: depends_on:
- broker - broker
@ -187,8 +178,6 @@ services:
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
- mysqldata:/var/lib/mysql - mysqldata:/var/lib/mysql
mysql-setup: mysql-setup:
labels:
datahub_setup_job: true
container_name: mysql-setup container_name: mysql-setup
depends_on: depends_on:
- mysql - mysql

240
docker/quickstart/docker-compose-without-neo4j.quickstart.yml
View File

@ -5,115 +5,115 @@ services:
broker: broker:
container_name: broker container_name: broker
depends_on: depends_on:
- zookeeper - zookeeper
environment: environment:
- KAFKA_BROKER_ID=1 - KAFKA_BROKER_ID=1
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
hostname: broker hostname: broker
image: confluentinc/cp-kafka:7.2.2 image: confluentinc/cp-kafka:7.2.2
ports: ports:
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
datahub-actions: datahub-actions:
depends_on: depends_on:
- datahub-gms - datahub-gms
environment: environment:
- DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_HOST=datahub-gms
- DATAHUB_GMS_PORT=8080 - DATAHUB_GMS_PORT=8080
- DATAHUB_GMS_PROTOCOL=http - DATAHUB_GMS_PROTOCOL=http
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
- SCHEMA_REGISTRY_URL=http://schema-registry:8081 - SCHEMA_REGISTRY_URL=http://schema-registry:8081
hostname: actions hostname: actions
image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} image: acryldata/datahub-actions:${ACTIONS_VERSION:-head}
restart: on-failure:5 restart: on-failure:5
datahub-frontend-react: datahub-frontend-react:
container_name: datahub-frontend-react container_name: datahub-frontend-react
depends_on: depends_on:
- datahub-gms - datahub-gms
environment: environment:
- DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_HOST=datahub-gms
- DATAHUB_GMS_PORT=8080 - DATAHUB_GMS_PORT=8080
- DATAHUB_SECRET=YouKnowNothing - DATAHUB_SECRET=YouKnowNothing
- DATAHUB_APP_VERSION=1.0 - DATAHUB_APP_VERSION=1.0
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
- ELASTIC_CLIENT_HOST=elasticsearch - ELASTIC_CLIENT_HOST=elasticsearch
- ELASTIC_CLIENT_PORT=9200 - ELASTIC_CLIENT_PORT=9200
hostname: datahub-frontend-react hostname: datahub-frontend-react
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
ports: ports:
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
volumes: volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins - ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-gms: datahub-gms:
container_name: datahub-gms container_name: datahub-gms
depends_on: depends_on:
- mysql - mysql
environment: environment:
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
- DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
- EBEAN_DATASOURCE_HOST=mysql:3306 - EBEAN_DATASOURCE_HOST=mysql:3306
- EBEAN_DATASOURCE_PASSWORD=datahub - EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
- EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_USERNAME=datahub
- ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PORT=9200
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
- ENTITY_SERVICE_ENABLE_RETENTION=true - ENTITY_SERVICE_ENABLE_RETENTION=true
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true - GRAPH_SERVICE_DIFF_MODE_ENABLED=true
- GRAPH_SERVICE_IMPL=elasticsearch - GRAPH_SERVICE_IMPL=elasticsearch
- JAVA_OPTS=-Xms1g -Xmx1g - JAVA_OPTS=-Xms1g -Xmx1g
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
- MAE_CONSUMER_ENABLED=true - MAE_CONSUMER_ENABLED=true
- MCE_CONSUMER_ENABLED=true - MCE_CONSUMER_ENABLED=true
- PE_CONSUMER_ENABLED=true - PE_CONSUMER_ENABLED=true
- UI_INGESTION_ENABLED=true - UI_INGESTION_ENABLED=true
hostname: datahub-gms hostname: datahub-gms
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
ports: ports:
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
volumes: volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins - ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-upgrade: datahub-upgrade:
command: command:
- -u - -u
- SystemUpdate - SystemUpdate
container_name: datahub-upgrade container_name: datahub-upgrade
environment: environment:
- EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_USERNAME=datahub
- EBEAN_DATASOURCE_PASSWORD=datahub - EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_HOST=mysql:3306 - EBEAN_DATASOURCE_HOST=mysql:3306
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
- ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PORT=9200
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
- GRAPH_SERVICE_IMPL=elasticsearch - GRAPH_SERVICE_IMPL=elasticsearch
- DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_HOST=datahub-gms
- DATAHUB_GMS_PORT=8080 - DATAHUB_GMS_PORT=8080
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
hostname: datahub-upgrade hostname: datahub-upgrade
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
labels: labels:
@ -121,30 +121,30 @@ services:
elasticsearch: elasticsearch:
container_name: elasticsearch container_name: elasticsearch
environment: environment:
- discovery.type=single-node - discovery.type=single-node
- xpack.security.enabled=false - xpack.security.enabled=false
- ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true
healthcheck: healthcheck:
retries: 4 retries: 4
start_period: 2m start_period: 2m
test: test:
- CMD-SHELL - CMD-SHELL
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
hostname: elasticsearch hostname: elasticsearch
image: elasticsearch:7.10.1 image: elasticsearch:7.10.1
mem_limit: 1g mem_limit: 1g
ports: ports:
- ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
volumes: volumes:
- esdata:/usr/share/elasticsearch/data - esdata:/usr/share/elasticsearch/data
elasticsearch-setup: elasticsearch-setup:
container_name: elasticsearch-setup container_name: elasticsearch-setup
depends_on: depends_on:
- elasticsearch - elasticsearch
environment: environment:
- ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PORT=9200
- ELASTICSEARCH_PROTOCOL=http - ELASTICSEARCH_PROTOCOL=http
hostname: elasticsearch-setup hostname: elasticsearch-setup
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
labels: labels:
@ -152,12 +152,12 @@ services:
kafka-setup: kafka-setup:
container_name: kafka-setup container_name: kafka-setup
depends_on: depends_on:
- broker - broker
- schema-registry - schema-registry
environment: environment:
- DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false} - DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false}
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
hostname: kafka-setup hostname: kafka-setup
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
labels: labels:
@ -166,55 +166,55 @@ services:
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
container_name: mysql container_name: mysql
environment: environment:
- MYSQL_DATABASE=datahub - MYSQL_DATABASE=datahub
- MYSQL_USER=datahub - MYSQL_USER=datahub
- MYSQL_PASSWORD=datahub - MYSQL_PASSWORD=datahub
- MYSQL_ROOT_PASSWORD=datahub - MYSQL_ROOT_PASSWORD=datahub
hostname: mysql hostname: mysql
image: mysql:5.7 image: mysql:5.7
ports: ports:
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
volumes: volumes:
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
- mysqldata:/var/lib/mysql - mysqldata:/var/lib/mysql
mysql-setup: mysql-setup:
labels:
datahub_setup_job: true
container_name: mysql-setup container_name: mysql-setup
depends_on: depends_on:
- mysql - mysql
environment: environment:
- MYSQL_HOST=mysql - MYSQL_HOST=mysql
- MYSQL_PORT=3306 - MYSQL_PORT=3306
- MYSQL_USERNAME=datahub - MYSQL_USERNAME=datahub
- MYSQL_PASSWORD=datahub - MYSQL_PASSWORD=datahub
- DATAHUB_DB_NAME=datahub - DATAHUB_DB_NAME=datahub
hostname: mysql-setup hostname: mysql-setup
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head} image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head}
labels:
datahub_setup_job: true
schema-registry: schema-registry:
container_name: schema-registry container_name: schema-registry
depends_on: depends_on:
- broker - broker
environment: environment:
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry - SCHEMA_REGISTRY_HOST_NAME=schemaregistry
- SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
hostname: schema-registry hostname: schema-registry
image: confluentinc/cp-schema-registry:7.2.2 image: confluentinc/cp-schema-registry:7.2.2
ports: ports:
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
zookeeper: zookeeper:
container_name: zookeeper container_name: zookeeper
environment: environment:
- ZOOKEEPER_CLIENT_PORT=2181 - ZOOKEEPER_CLIENT_PORT=2181
- ZOOKEEPER_TICK_TIME=2000 - ZOOKEEPER_TICK_TIME=2000
hostname: zookeeper hostname: zookeeper
image: confluentinc/cp-zookeeper:7.2.2 image: confluentinc/cp-zookeeper:7.2.2
ports: ports:
- ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
volumes: volumes:
- zkdata:/var/lib/zookeeper - zkdata:/var/lib/zookeeper
version: "2.3" version: '2.3'
volumes: volumes:
esdata: null esdata: null
mysqldata: null mysqldata: null

50
docker/quickstart/docker-compose.monitoring.quickstart.yml
View File

@ -1,47 +1,47 @@
services: services:
datahub-frontend-react: datahub-frontend-react:
environment: environment:
- ENABLE_PROMETHEUS=true - ENABLE_PROMETHEUS=true
- ENABLE_OTEL=true - ENABLE_OTEL=true
- OTEL_TRACES_EXPORTER=jaeger - OTEL_TRACES_EXPORTER=jaeger
- OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
- OTEL_METRICS_EXPORTER=none - OTEL_METRICS_EXPORTER=none
- OTEL_SERVICE_NAME=datahub-gms - OTEL_SERVICE_NAME=datahub-gms
ports: ports:
- "4318" - '4318'
datahub-gms: datahub-gms:
environment: environment:
- ENABLE_PROMETHEUS=true - ENABLE_PROMETHEUS=true
- ENABLE_OTEL=true - ENABLE_OTEL=true
- OTEL_TRACES_EXPORTER=jaeger - OTEL_TRACES_EXPORTER=jaeger
- OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250 - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
- OTEL_METRICS_EXPORTER=none - OTEL_METRICS_EXPORTER=none
- OTEL_SERVICE_NAME=datahub-gms - OTEL_SERVICE_NAME=datahub-gms
ports: ports:
- "4318" - '4318'
grafana: grafana:
depends_on: depends_on:
- prometheus - prometheus
image: grafana/grafana:9.1.4 image: grafana/grafana:9.1.4
ports: ports:
- 3001:3000 - 3001:3000
volumes: volumes:
- grafana-storage:/var/lib/grafana - grafana-storage:/var/lib/grafana
- ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources - ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources
- ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards - ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards
jaeger-all-in-one: jaeger-all-in-one:
image: jaegertracing/all-in-one:latest image: jaegertracing/all-in-one:latest
ports: ports:
- 16686:16686 - 16686:16686
- "14268" - '14268'
- "14250" - '14250'
prometheus: prometheus:
container_name: prometheus container_name: prometheus
image: prom/prometheus:latest image: prom/prometheus:latest
ports: ports:
- 9089:9090 - 9089:9090
volumes: volumes:
- ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml - ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml
version: "2.3" version: '2.3'
volumes: volumes:
grafana-storage: null grafana-storage: null

264
docker/quickstart/docker-compose.quickstart.yml
View File

@ -5,124 +5,124 @@ services:
broker: broker:
container_name: broker container_name: broker
depends_on: depends_on:
- zookeeper - zookeeper
environment: environment:
- KAFKA_BROKER_ID=1 - KAFKA_BROKER_ID=1
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
hostname: broker hostname: broker
image: confluentinc/cp-kafka:7.2.2 image: confluentinc/cp-kafka:7.2.2
ports: ports:
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
volumes: volumes:
- broker:/var/lib/kafka/data/ - broker:/var/lib/kafka/data/
datahub-actions: datahub-actions:
depends_on: depends_on:
- datahub-gms - datahub-gms
environment: environment:
- DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_HOST=datahub-gms
- DATAHUB_GMS_PORT=8080 - DATAHUB_GMS_PORT=8080
- DATAHUB_GMS_PROTOCOL=http - DATAHUB_GMS_PROTOCOL=http
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
- DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
- METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
- METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
- SCHEMA_REGISTRY_URL=http://schema-registry:8081 - SCHEMA_REGISTRY_URL=http://schema-registry:8081
hostname: actions hostname: actions
image: acryldata/datahub-actions:${ACTIONS_VERSION:-head} image: acryldata/datahub-actions:${ACTIONS_VERSION:-head}
restart: on-failure:5 restart: on-failure:5
datahub-frontend-react: datahub-frontend-react:
container_name: datahub-frontend-react container_name: datahub-frontend-react
depends_on: depends_on:
- datahub-gms - datahub-gms
environment: environment:
- DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_HOST=datahub-gms
- DATAHUB_GMS_PORT=8080 - DATAHUB_GMS_PORT=8080
- DATAHUB_SECRET=YouKnowNothing - DATAHUB_SECRET=YouKnowNothing
- DATAHUB_APP_VERSION=1.0 - DATAHUB_APP_VERSION=1.0
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
- ELASTIC_CLIENT_HOST=elasticsearch - ELASTIC_CLIENT_HOST=elasticsearch
- ELASTIC_CLIENT_PORT=9200 - ELASTIC_CLIENT_PORT=9200
hostname: datahub-frontend-react hostname: datahub-frontend-react
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
ports: ports:
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
volumes: volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins - ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-gms: datahub-gms:
container_name: datahub-gms container_name: datahub-gms
depends_on: depends_on:
- mysql - mysql
- neo4j - neo4j
environment: environment:
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
- DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
- EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_USERNAME=datahub
- EBEAN_DATASOURCE_PASSWORD=datahub - EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_HOST=mysql:3306 - EBEAN_DATASOURCE_HOST=mysql:3306
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2 - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
- ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PORT=9200
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
- NEO4J_HOST=http://neo4j:7474 - NEO4J_HOST=http://neo4j:7474
- NEO4J_URI=bolt://neo4j - NEO4J_URI=bolt://neo4j
- NEO4J_USERNAME=neo4j - NEO4J_USERNAME=neo4j
- NEO4J_PASSWORD=datahub - NEO4J_PASSWORD=datahub
- JAVA_OPTS=-Xms1g -Xmx1g - JAVA_OPTS=-Xms1g -Xmx1g
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true - GRAPH_SERVICE_DIFF_MODE_ENABLED=true
- GRAPH_SERVICE_IMPL=neo4j - GRAPH_SERVICE_IMPL=neo4j
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
- ENTITY_SERVICE_ENABLE_RETENTION=true - ENTITY_SERVICE_ENABLE_RETENTION=true
- MAE_CONSUMER_ENABLED=true - MAE_CONSUMER_ENABLED=true
- MCE_CONSUMER_ENABLED=true - MCE_CONSUMER_ENABLED=true
- PE_CONSUMER_ENABLED=true - PE_CONSUMER_ENABLED=true
- UI_INGESTION_ENABLED=true - UI_INGESTION_ENABLED=true
- METADATA_SERVICE_AUTH_ENABLED=false - METADATA_SERVICE_AUTH_ENABLED=false
hostname: datahub-gms hostname: datahub-gms
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
ports: ports:
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
volumes: volumes:
- ${HOME}/.datahub/plugins/:/etc/datahub/plugins - ${HOME}/.datahub/plugins/:/etc/datahub/plugins
- ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources - ${HOME}/.datahub/plugins/auth/resources/:/etc/datahub/plugins/auth/resources
datahub-upgrade: datahub-upgrade:
command: command:
- -u - -u
- SystemUpdate - SystemUpdate
container_name: datahub-upgrade container_name: datahub-upgrade
environment: environment:
- EBEAN_DATASOURCE_USERNAME=datahub - EBEAN_DATASOURCE_USERNAME=datahub
- EBEAN_DATASOURCE_PASSWORD=datahub - EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_HOST=mysql:3306 - EBEAN_DATASOURCE_HOST=mysql:3306
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
- ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PORT=9200
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
- GRAPH_SERVICE_IMPL=elasticsearch - GRAPH_SERVICE_IMPL=elasticsearch
- DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_HOST=datahub-gms
- DATAHUB_GMS_PORT=8080 - DATAHUB_GMS_PORT=8080
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
hostname: datahub-upgrade hostname: datahub-upgrade
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
labels: labels:
@ -130,30 +130,30 @@ services:
elasticsearch: elasticsearch:
container_name: elasticsearch container_name: elasticsearch
environment: environment:
- discovery.type=single-node - discovery.type=single-node
- xpack.security.enabled=false - xpack.security.enabled=false
- ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true
healthcheck: healthcheck:
retries: 4 retries: 4
start_period: 2m start_period: 2m
test: test:
- CMD-SHELL - CMD-SHELL
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1 - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
hostname: elasticsearch hostname: elasticsearch
image: elasticsearch:7.10.1 image: elasticsearch:7.10.1
mem_limit: 1g mem_limit: 1g
ports: ports:
- ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
volumes: volumes:
- esdata:/usr/share/elasticsearch/data - esdata:/usr/share/elasticsearch/data
elasticsearch-setup: elasticsearch-setup:
container_name: elasticsearch-setup container_name: elasticsearch-setup
depends_on: depends_on:
- elasticsearch - elasticsearch
environment: environment:
- ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PORT=9200
- ELASTICSEARCH_PROTOCOL=http - ELASTICSEARCH_PROTOCOL=http
hostname: elasticsearch-setup hostname: elasticsearch-setup
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
labels: labels:
@ -161,12 +161,12 @@ services:
kafka-setup: kafka-setup:
container_name: kafka-setup container_name: kafka-setup
depends_on: depends_on:
- broker - broker
- schema-registry - schema-registry
environment: environment:
- DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false} - DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false}
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
hostname: kafka-setup hostname: kafka-setup
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
labels: labels:
@ -175,27 +175,27 @@ services:
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
container_name: mysql container_name: mysql
environment: environment:
- MYSQL_DATABASE=datahub - MYSQL_DATABASE=datahub
- MYSQL_USER=datahub - MYSQL_USER=datahub
- MYSQL_PASSWORD=datahub - MYSQL_PASSWORD=datahub
- MYSQL_ROOT_PASSWORD=datahub - MYSQL_ROOT_PASSWORD=datahub
hostname: mysql hostname: mysql
image: mysql:5.7 image: mysql:5.7
ports: ports:
- ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
volumes: volumes:
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
- mysqldata:/var/lib/mysql - mysqldata:/var/lib/mysql
mysql-setup: mysql-setup:
container_name: mysql-setup container_name: mysql-setup
depends_on: depends_on:
- mysql - mysql
environment: environment:
- MYSQL_HOST=mysql - MYSQL_HOST=mysql
- MYSQL_PORT=3306 - MYSQL_PORT=3306
- MYSQL_USERNAME=datahub - MYSQL_USERNAME=datahub
- MYSQL_PASSWORD=datahub - MYSQL_PASSWORD=datahub
- DATAHUB_DB_NAME=datahub - DATAHUB_DB_NAME=datahub
hostname: mysql-setup hostname: mysql-setup
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head} image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head}
labels: labels:
@ -203,40 +203,40 @@ services:
neo4j: neo4j:
container_name: neo4j container_name: neo4j
environment: environment:
- NEO4J_AUTH=neo4j/datahub - NEO4J_AUTH=neo4j/datahub
- NEO4J_dbms_default__database=graph.db - NEO4J_dbms_default__database=graph.db
- NEO4J_dbms_allow__upgrade=true - NEO4J_dbms_allow__upgrade=true
hostname: neo4j hostname: neo4j
image: neo4j:4.4.9-community image: neo4j:4.4.9-community
ports: ports:
- ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474 - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474
- ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687 - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687
volumes: volumes:
- neo4jdata:/data - neo4jdata:/data
schema-registry: schema-registry:
container_name: schema-registry container_name: schema-registry
depends_on: depends_on:
- broker - broker
environment: environment:
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry - SCHEMA_REGISTRY_HOST_NAME=schemaregistry
- SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
- SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092
hostname: schema-registry hostname: schema-registry
image: confluentinc/cp-schema-registry:7.2.2 image: confluentinc/cp-schema-registry:7.2.2
ports: ports:
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
zookeeper: zookeeper:
container_name: zookeeper container_name: zookeeper
environment: environment:
- ZOOKEEPER_CLIENT_PORT=2181 - ZOOKEEPER_CLIENT_PORT=2181
- ZOOKEEPER_TICK_TIME=2000 - ZOOKEEPER_TICK_TIME=2000
hostname: zookeeper hostname: zookeeper
image: confluentinc/cp-zookeeper:7.2.2 image: confluentinc/cp-zookeeper:7.2.2
ports: ports:
- ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
volumes: volumes:
- zkdata:/var/lib/zookeeper - zkdata:/var/lib/zookeeper
version: "2.3" version: '2.3'
volumes: volumes:
broker: null broker: null
esdata: null esdata: null

22
docker/quickstart/generate_and_compare.sh
View File

@ -1,8 +1,5 @@
#!/bin/bash #!/bin/bash
# this scripts checks if docker-compose$flavour.quickstart.yml is up to date for these 'flavours':
FLAVOURS=("" "-without-neo4j" ".monitoring")
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR" cd "$DIR"
@ -12,21 +9,4 @@ python3 -m venv venv
source venv/bin/activate source venv/bin/activate
pip install -r requirements.txt pip install -r requirements.txt
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml temp.quickstart.yml python generate_docker_quickstart.py check-all
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml temp-without-neo4j.quickstart.yml
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose.m1.yml temp-m1.quickstart.yml
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml ../docker-compose-without-neo4j.m1.yml temp-without-neo4j-m1.quickstart.yml
python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.monitoring.quickstart.yml
python generate_docker_quickstart.py ../docker-compose.consumers.yml temp.consumers.quickstart.yml
python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml temp.consumers-without-neo4j.quickstart.yml
for flavour in "${FLAVOURS[@]}"
do
if cmp <(yq -i -P 'sort_keys(..)' docker-compose$flavour.quickstart.yml) <(yq -i -P 'sort_keys(..)' temp$flavour.quickstart.yml); then
echo "docker-compose$flavour.quickstart.yml is up to date."
else
echo "docker-compose$flavour.quickstart.yml is out of date."
exit 1
fi
done

174
docker/quickstart/generate_docker_quickstart.py
View File

@ -1,4 +1,8 @@
from io import StringIO
from typing import List
import os import os
import sys
import pathlib
from collections.abc import Mapping from collections.abc import Mapping
import click import click
@ -6,8 +10,38 @@ import yaml
from dotenv import dotenv_values from dotenv import dotenv_values
from yaml import Loader from yaml import Loader
# Generates a merged docker-compose file with env variables inlined. COMPOSE_SPECS = {
# Usage: python3 docker_compose_cli_gen.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose-gen.yml "docker-compose.quickstart.yml": [
"../docker-compose.yml",
"../docker-compose.override.yml",
],
"docker-compose-m1.quickstart.yml": [
"../docker-compose.yml",
"../docker-compose.override.yml",
"../docker-compose.m1.yml",
],
"docker-compose-without-neo4j.quickstart.yml": [
"../docker-compose-without-neo4j.yml",
"../docker-compose-without-neo4j.override.yml",
],
"docker-compose-without-neo4j-m1.quickstart.yml": [
"../docker-compose-without-neo4j.yml",
"../docker-compose-without-neo4j.override.yml",
"../docker-compose-without-neo4j.m1.yml",
],
"docker-compose.monitoring.quickstart.yml": [
"../monitoring/docker-compose.monitoring.yml",
],
"docker-compose.consumers.quickstart.yml": [
"../docker-compose.consumers.yml",
],
"docker-compose.consumers-without-neo4j.quickstart.yml": [
"../docker-compose.consumers-without-neo4j.yml",
],
"docker-compose.kafka-setup.quickstart.yml": [
"../docker-compose.kafka-setup.yml",
],
}
omitted_services = [ omitted_services = [
"kafka-rest-proxy", "kafka-rest-proxy",
@ -34,6 +68,7 @@ def dict_merge(dct, merge_dct):
else: else:
dct[k] = merge_dct[k] dct[k] = merge_dct[k]
def modify_docker_config(base_path, docker_yaml_config): def modify_docker_config(base_path, docker_yaml_config):
if not docker_yaml_config["services"]: if not docker_yaml_config["services"]:
docker_yaml_config["services"] = {} docker_yaml_config["services"] = {}
@ -91,17 +126,35 @@ def modify_docker_config(base_path, docker_yaml_config):
docker_yaml_config["version"] = "2.3" docker_yaml_config["version"] = "2.3"
@click.command() def dedup_env_vars(merged_docker_config):
@click.argument( for service in merged_docker_config["services"]:
"compose-files", if "environment" in merged_docker_config["services"][service]:
nargs=-1, lst = merged_docker_config["services"][service]["environment"]
type=click.Path( if lst is not None:
exists=True, # use a set to cache duplicates
dir_okay=False, caches = set()
), results = {}
) for item in lst:
@click.argument("output-file", type=click.Path()) partitions = item.rpartition("=")
def generate(compose_files, output_file) -> None: prefix = partitions[0]
suffix = partitions[1]
# check whether prefix already exists
if prefix not in caches and suffix != "":
results[prefix] = item
caches.add(prefix)
if set(lst) != set([v for k, v in results.items()]):
sorted_vars = sorted([k for k in results])
merged_docker_config["services"][service]["environment"] = [
results[var] for var in sorted_vars
]
def merge_files(compose_files: List[str]) -> str:
"""
Generates a merged docker-compose file with env variables inlined.
Example Usage: python3 generate_docker_quickstart.py generate-one ../docker-compose.yml ../docker-compose.override.yml ../docker-compose-gen.yml
"""
# Resolve .env files to inlined vars # Resolve .env files to inlined vars
modified_files = [] modified_files = []
@ -121,40 +174,77 @@ def generate(compose_files, output_file) -> None:
# Dedup env vars, last wins # Dedup env vars, last wins
dedup_env_vars(merged_docker_config) dedup_env_vars(merged_docker_config)
# Generate yaml to string.
out = StringIO()
yaml.dump(
merged_docker_config,
out,
default_flow_style=False,
width=1000,
)
return out.getvalue()
@click.group()
def main_cmd() -> None:
pass
@main_cmd.command()
@click.argument(
"compose-files",
nargs=-1,
type=click.Path(
exists=True,
dir_okay=False,
),
)
@click.argument("output-file", type=click.Path())
def generate_one(compose_files, output_file) -> None:
"""
Generates a merged docker-compose file with env variables inlined.
Example Usage: python3 generate_docker_quickstart.py generate-one ../docker-compose.yml ../docker-compose.override.yml ../docker-compose-gen.yml
"""
merged_contents = merge_files(compose_files)
# Write output file # Write output file
output_dir = os.path.dirname(output_file) pathlib.Path(output_file).parent.mkdir(parents=True, exist_ok=True)
if len(output_dir) and not os.path.exists(output_dir): pathlib.Path(output_file).write_text(merged_contents)
os.makedirs(output_dir)
with open(output_file, "w") as new_conf_file:
yaml.dump(
merged_docker_config,
new_conf_file,
default_flow_style=False,
)
print(f"Successfully generated {output_file}.") print(f"Successfully generated {output_file}.")
def dedup_env_vars(merged_docker_config): @main_cmd.command()
for service in merged_docker_config['services']: @click.pass_context
if 'environment' in merged_docker_config['services'][service]: def generate_all(ctx: click.Context) -> None:
lst = merged_docker_config['services'][service]['environment'] """
if lst is not None: Generates all merged docker-compose files with env variables inlined.
# use a set to cache duplicates """
caches = set()
results = {} for output_compose_file, inputs in COMPOSE_SPECS.items():
for item in lst: ctx.invoke(generate_one, compose_files=inputs, output_file=output_compose_file)
partitions = item.rpartition('=')
prefix = partitions[0]
suffix = partitions[1] @main_cmd.command()
# check whether prefix already exists def check_all() -> None:
if prefix not in caches and suffix != "": """
results[prefix] = item Checks that the generated docker-compose files are up to date.
caches.add(prefix) """
if set(lst) != set([v for k,v in results.items()]):
sorted_vars = sorted([k for k in results]) for output_compose_file, inputs in COMPOSE_SPECS.items():
merged_docker_config['services'][service]['environment'] = [results[var] for var in sorted_vars] expected = merge_files(inputs)
# Check that the files match.
current = pathlib.Path(output_compose_file).read_text()
if expected != current:
print(
f"File {output_compose_file} is out of date. Please run `python3 generate_docker_quickstart.py generate-all`."
)
sys.exit(1)
if __name__ == "__main__": if __name__ == "__main__":
generate() main_cmd()

9
docker/quickstart/generate_docker_quickstart.sh
View File

@ -9,11 +9,4 @@ python3 -m venv venv
source venv/bin/activate source venv/bin/activate
pip install -r requirements.txt pip install -r requirements.txt
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml docker-compose.quickstart.yml python generate_docker_quickstart.py generate-all
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml docker-compose-without-neo4j.quickstart.yml
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml ../docker-compose.m1.yml docker-compose-m1.quickstart.yml
python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml ../docker-compose-without-neo4j.m1.yml docker-compose-without-neo4j-m1.quickstart.yml
python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.monitoring.quickstart.yml
python generate_docker_quickstart.py ../docker-compose.consumers.yml docker-compose.consumers.quickstart.yml
python generate_docker_quickstart.py ../docker-compose.consumers-without-neo4j.yml docker-compose.consumers-without-neo4j.quickstart.yml
python generate_docker_quickstart.py ../docker-compose.kafka-setup.yml docker-compose.kafka-setup.quickstart.yml