yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-25 17:08:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(quickstart): remove container_name from compose files (#9809)

Browse Source
This commit is contained in:
Harshal Sheth 2024-02-09 20:45:30 -05:00 committed by GitHub
parent 7d73c41989
commit 5793467e29
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 3 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docker/cassandra/docker-compose.cassandra.yml
View File

@ -3,7 +3,6 @@
version: '3.8'
services:
cassandra:
container_name: cassandra
hostname: cassandra
image: cassandra:3.11
ports:
@ -16,7 +15,6 @@ services:
volumes:
- cassandradata:/var/lib/cassandra
cassandra-load-keyspace:
container_name: cassandra-setup
image: cassandra:3.11
depends_on:
cassandra:

12
docker/docker-compose-with-cassandra.yml
View File

@ -7,7 +7,6 @@
version: '3.9'
services:
datahub-frontend-react:
container_name: datahub-frontend-react
hostname: datahub-frontend-react
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
ports:
@ -22,7 +21,6 @@ services:
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-actions:
container_name: datahub-actions
hostname: actions
image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head}
env_file: datahub-actions/env/docker.env
@ -33,7 +31,6 @@ services:
datahub-gms:
condition: service_healthy
datahub-gms:
container_name: datahub-gms
hostname: datahub-gms
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
ports:
@ -54,7 +51,6 @@ services:
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-upgrade:
container_name: datahub-upgrade
hostname: datahub-upgrade
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
command:
@ -76,7 +72,6 @@ services:
schema-registry:
condition: service_healthy
cassandra-setup:
container_name: cassandra-setup
hostname: cassandra-setup
image: cassandra:3.11
command: /bin/bash -c "cqlsh cassandra -f /init.cql"
@ -89,7 +84,6 @@ services:
datahub_setup_job: true
# This "container" is a workaround to pre-create search indices
elasticsearch-setup:
container_name: elasticsearch-setup
hostname: elasticsearch-setup
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
build:
@ -105,7 +99,6 @@ services:
labels:
datahub_setup_job: true
cassandra:
container_name: cassandra
hostname: cassandra
image: cassandra:3.11
ports:
@ -118,7 +111,6 @@ services:
volumes:
- cassandradata:/var/lib/cassandra
elasticsearch:
container_name: elasticsearch
hostname: elasticsearch
image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1}
ports:
@ -136,7 +128,6 @@ services:
volumes:
- esdata:/usr/share/elasticsearch/data
neo4j:
container_name: neo4j
hostname: neo4j
image: neo4j:4.0.6
ports:
@ -152,7 +143,6 @@ services:
volumes:
- neo4jdata:/data
schema-registry:
container_name: schema-registry
hostname: schema-registry
image: confluentinc/cp-schema-registry:7.4.0
ports:
@ -168,7 +158,6 @@ services:
broker:
condition: service_healthy
broker:
container_name: broker
hostname: broker
image: confluentinc/cp-kafka:7.4.0
ports:
@ -187,7 +176,6 @@ services:
volumes:
- broker:/var/lib/kafka/data/
zookeeper:
container_name: zookeeper
hostname: zookeeper
image: confluentinc/cp-zookeeper:7.4.0
ports:

3
docker/docker-compose-without-neo4j.override.yml
View File

@ -12,7 +12,6 @@ services:
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-upgrade:
container_name: datahub-upgrade
hostname: datahub-upgrade
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
command:
@ -30,7 +29,6 @@ services:
kafka-setup:
condition: service_completed_successfully
mysql-setup:
container_name: mysql-setup
hostname: mysql-setup
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head}
build:
@ -46,7 +44,6 @@ services:
environment:
- DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false}
mysql:
container_name: mysql
hostname: mysql
image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password

3
docker/docker-compose-without-neo4j.postgres.override.yml
View File

@ -16,7 +16,6 @@ services:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-upgrade:
container_name: datahub-upgrade
hostname: datahub-upgrade
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
command:
@ -37,7 +36,6 @@ services:
condition: service_completed_successfully
postgres-setup:
container_name: postgres-setup
hostname: postgres-setup
image: ${DATAHUB_POSTGRES_SETUP_IMAGE:-acryldata/datahub-postgres-setup}:${DATAHUB_VERSION:-head}
build:
@ -51,7 +49,6 @@ services:
datahub_setup_job: true
postgres:
container_name: postgres
hostname: postgres
image: postgres:${DATAHUB_POSTGRES_VERSION:-12.3}
env_file: postgres/env/docker.env

10
docker/docker-compose-without-neo4j.yml
View File

@ -7,7 +7,6 @@
version: '3.9'
services:
datahub-frontend-react:
container_name: datahub-frontend-react
hostname: datahub-frontend-react
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
ports:
@ -23,7 +22,6 @@ services:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-actions:
container_name: datahub-actions
hostname: actions
image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head}
env_file: datahub-actions/env/docker.env
@ -34,7 +32,6 @@ services:
datahub-gms:
condition: service_healthy
datahub-gms:
container_name: datahub-gms
hostname: datahub-gms
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
ports:
@ -57,7 +54,6 @@ services:
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-upgrade:
container_name: datahub-upgrade
hostname: datahub-upgrade
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
command:
@ -76,7 +72,6 @@ services:
datahub_setup_job: true
# This "container" is a workaround to pre-create search indices
elasticsearch-setup:
container_name: elasticsearch-setup
hostname: elasticsearch-setup
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
build:
@ -92,7 +87,6 @@ services:
labels:
datahub_setup_job: true
kafka-setup:
container_name: kafka-setup
hostname: kafka-setup
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
build:
@ -107,7 +101,6 @@ services:
labels:
datahub_setup_job: true
elasticsearch:
container_name: elasticsearch
hostname: elasticsearch
image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1}
ports:
@ -129,7 +122,6 @@ services:
volumes:
- esdata:/usr/share/elasticsearch/data
schema-registry:
container_name: schema-registry
hostname: schema-registry
image: confluentinc/cp-schema-registry:7.4.0
ports:
@ -145,7 +137,6 @@ services:
broker:
condition: service_healthy
broker:
container_name: broker
hostname: broker
image: confluentinc/cp-kafka:7.4.0
ports:
@ -163,7 +154,6 @@ services:
volumes:
- broker:/var/lib/kafka/data/
zookeeper:
container_name: zookeeper
hostname: zookeeper
image: confluentinc/cp-zookeeper:7.4.0
ports:

2
docker/docker-compose.consumers-without-neo4j.yml
View File

@ -6,7 +6,6 @@ services:
- MAE_CONSUMER_ENABLED=false
- MCE_CONSUMER_ENABLED=false
datahub-mae-consumer:
container_name: datahub-mae-consumer
hostname: datahub-mae-consumer
image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head}
ports:
@ -19,7 +18,6 @@ services:
- KAFKA_CONSUMER_STOP_ON_DESERIALIZATION_ERROR=${KAFKA_CONSUMER_STOP_ON_DESERIALIZATION_ERROR:-true}
- KAFKA_CONSUMER_HEALTH_CHECK_ENABLED=${KAFKA_CONSUMER_HEALTH_CHECK_ENABLED:-true}
datahub-mce-consumer:
container_name: datahub-mce-consumer
hostname: datahub-mce-consumer
image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head}
ports:

2
docker/docker-compose.consumers.yml
View File

@ -6,7 +6,6 @@ services:
- MAE_CONSUMER_ENABLED=false
- MCE_CONSUMER_ENABLED=false
datahub-mae-consumer:
container_name: datahub-mae-consumer
hostname: datahub-mae-consumer
image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head}
ports:
@ -22,7 +21,6 @@ services:
neo4j:
condition: service_healthy
datahub-mce-consumer:
container_name: datahub-mce-consumer
hostname: datahub-mce-consumer
image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head}
ports:

2
docker/docker-compose.override.yml
View File

@ -14,7 +14,6 @@ services:
environment:
- GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
mysql-setup:
container_name: mysql-setup
hostname: mysql-setup
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head}
build:
@ -30,7 +29,6 @@ services:
environment:
- DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false}
mysql:
container_name: mysql
hostname: mysql
image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password

3
docker/docker-compose.tools.yml
View File

@ -6,7 +6,6 @@ services:
image: confluentinc/cp-kafka-rest:7.4.0
env_file: kafka-rest-proxy/env/docker.env
hostname: kafka-rest-proxy
container_name: kafka-rest-proxy
ports:
- "8082:8082"
depends_on:
@ -18,7 +17,6 @@ services:
image: landoop/kafka-topics-ui:0.9.4
env_file: kafka-topics-ui/env/docker.env
hostname: kafka-topics-ui
container_name: kafka-topics-ui
ports:
- "18000:8000"
depends_on:
@ -30,7 +28,6 @@ services:
kibana:
image: kibana:7.10.1
env_file: kibana/env/docker.env
container_name: kibana
hostname: kibana
ports:
- "5601:5601"

11
docker/docker-compose.yml
View File

@ -7,7 +7,6 @@
version: '3.9'
services:
datahub-frontend-react:
container_name: datahub-frontend-react
hostname: datahub-frontend-react
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
ports:
@ -22,7 +21,6 @@ services:
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-actions:
container_name: datahub-actions
hostname: actions
image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head}
env_file: datahub-actions/env/docker.env
@ -33,7 +31,6 @@ services:
datahub-gms:
condition: service_healthy
datahub-gms:
container_name: datahub-gms
hostname: datahub-gms
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
environment:
@ -55,7 +52,6 @@ services:
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-upgrade:
container_name: datahub-upgrade
hostname: datahub-upgrade
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
command:
@ -78,7 +74,6 @@ services:
condition: service_healthy
# This "container" is a workaround to pre-create search indices
elasticsearch-setup:
container_name: elasticsearch-setup
hostname: elasticsearch-setup
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
build:
@ -97,7 +92,6 @@ services:
# This is not required in most cases, kept here for backwards compatibility with older clients that
# explicitly wait for this container
kafka-setup:
container_name: kafka-setup
hostname: kafka-setup
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
build:
@ -112,7 +106,6 @@ services:
labels:
datahub_setup_job: true
elasticsearch:
container_name: elasticsearch
hostname: elasticsearch
image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1}
ports:
@ -134,7 +127,6 @@ services:
volumes:
- esdata:/usr/share/elasticsearch/data
neo4j:
container_name: neo4j
hostname: neo4j
image: neo4j:4.4.9-community
ports:
@ -150,7 +142,6 @@ services:
volumes:
- neo4jdata:/data
schema-registry:
container_name: schema-registry
hostname: schema-registry
image: confluentinc/cp-schema-registry:7.4.0
ports:
@ -166,7 +157,6 @@ services:
broker:
condition: service_healthy
broker:
container_name: broker
hostname: broker
image: confluentinc/cp-kafka:7.4.0
ports:
@ -184,7 +174,6 @@ services:
volumes:
- broker:/var/lib/kafka/data/
zookeeper:
container_name: zookeeper
hostname: zookeeper
image: confluentinc/cp-zookeeper:7.4.0
ports:

1
docker/ingestion/docker-compose.yml
View File

@ -7,7 +7,6 @@ services:
dockerfile: docker/datahub-ingestion/Dockerfile
image: linkedin/datahub-ingestion:${DATAHUB_VERSION:-head}
hostname: ingestion
container_name: ingestion
command: "ingest -c /sample_recipe.yml"
volumes:
# Most of the config is embedded inside the sample recipe file.

1
docker/mariadb/docker-compose.mariadb.yml
View File

@ -3,7 +3,6 @@
version: '3.8'
services:
mariadb:
container_name: mariadb
hostname: mariadb
image: mariadb:10.5
env_file: env/docker.env

1
docker/monitoring/docker-compose.monitoring.yml
View File

@ -38,7 +38,6 @@ services:
- '14268'
- '14250'
prometheus:
container_name: prometheus
image: prom/prometheus:latest
volumes:
- ./monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml

1
docker/mysql/docker-compose.mysql.yml
View File

@ -3,7 +3,6 @@
version: '3.8'
services:
mysql:
container_name: mysql
hostname: mysql
image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
env_file: env/docker.env

10
docker/profiles/docker-compose.actions.yml
View File

@ -15,7 +15,6 @@ x-datahub-actions-service-dev: &datahub-actions-service-dev
services:
datahub-actions-quickstart:
<<: *datahub-actions-service
container_name: actions
profiles:
- quickstart
- quickstart-backend
@ -24,7 +23,6 @@ services:
condition: service_healthy
datahub-actions-quickstart-cassandra:
<<: *datahub-actions-service
container_name: actions
profiles:
- quickstart-cassandra
depends_on:
@ -32,7 +30,6 @@ services:
condition: service_healthy
datahub-actions-quickstart-postgres:
<<: *datahub-actions-service
container_name: actions
profiles:
- quickstart-postgres
depends_on:
@ -40,7 +37,6 @@ services:
condition: service_healthy
datahub-actions-quickstart-consumers:
<<: *datahub-actions-service
container_name: actions
profiles:
- quickstart-consumers
depends_on:
@ -48,7 +44,6 @@ services:
condition: service_healthy
datahub-actions-debug:
<<: *datahub-actions-service-dev
container_name: actions-dev
profiles:
- debug
- debug-backend
@ -57,7 +52,6 @@ services:
condition: service_healthy
datahub-actions-debug-postgres:
<<: *datahub-actions-service-dev
container_name: actions-dev
profiles:
- debug-postgres
depends_on:
@ -65,7 +59,6 @@ services:
condition: service_healthy
datahub-actions-debug-cassandra:
<<: *datahub-actions-service-dev
container_name: actions-dev
profiles:
- debug-cassandra
depends_on:
@ -73,7 +66,6 @@ services:
condition: service_healthy
datahub-actions-debug-consumers:
<<: *datahub-actions-service-dev
container_name: actions-dev
profiles:
- debug-consumers
depends_on:
@ -81,7 +73,6 @@ services:
condition: service_healthy
datahub-actions-debug-neo4j:
<<: *datahub-actions-service-dev
container_name: actions-dev
profiles:
- debug-neo4j
depends_on:
@ -89,7 +80,6 @@ services:
condition: service_healthy
datahub-actions-debug-elasticsearch:
<<: *datahub-actions-service-dev
container_name: actions-dev
profiles:
- debug-elasticsearch
depends_on:

11
docker/profiles/docker-compose.frontend.yml
View File

@ -26,7 +26,6 @@ x-datahub-frontend-service-dev: &datahub-frontend-service-dev
services:
frontend-quickstart:
<<: *datahub-frontend-service
container_name: frontend
profiles:
- quickstart
- quickstart-frontend
@ -35,7 +34,6 @@ services:
condition: service_completed_successfully
frontend-quickstart-cassandra:
<<: *datahub-frontend-service
container_name: frontend
profiles:
- quickstart-cassandra
depends_on:
@ -43,7 +41,6 @@ services:
condition: service_completed_successfully
frontend-quickstart-postgres:
<<: *datahub-frontend-service
container_name: frontend
profiles:
- quickstart-postgres
depends_on:
@ -51,7 +48,6 @@ services:
condition: service_completed_successfully
frontend-quickstart-consumers:
<<: *datahub-frontend-service
container_name: frontend
profiles:
- quickstart-consumers
depends_on:
@ -59,7 +55,6 @@ services:
condition: service_completed_successfully
frontend-debug:
<<: *datahub-frontend-service-dev
container_name: datahub-frontend-dev
profiles:
- debug
depends_on:
@ -67,7 +62,6 @@ services:
condition: service_completed_successfully
frontend-debug-frontend:
<<: *datahub-frontend-service-dev
container_name: datahub-frontend-dev
profiles:
- debug-frontend
depends_on:
@ -79,7 +73,6 @@ services:
condition: service_completed_successfully
frontend-debug-postgres:
<<: *datahub-frontend-service-dev
container_name: datahub-frontend-dev
profiles:
- debug-postgres
depends_on:
@ -87,7 +80,6 @@ services:
condition: service_completed_successfully
frontend-debug-cassandra:
<<: *datahub-frontend-service-dev
container_name: datahub-frontend-dev
profiles:
- debug-cassandra
depends_on:
@ -95,7 +87,6 @@ services:
condition: service_completed_successfully
frontend-debug-consumers:
<<: *datahub-frontend-service-dev
container_name: datahub-frontend-dev
profiles:
- debug-consumers
depends_on:
@ -103,7 +94,6 @@ services:
condition: service_completed_successfully
frontend-debug-neo4j:
<<: *datahub-frontend-service-dev
container_name: datahub-frontend-dev
profiles:
- debug-neo4j
depends_on:
@ -111,7 +101,6 @@ services:
condition: service_completed_successfully
frontend-debug-elasticsearch:
<<: *datahub-frontend-service-dev
container_name: datahub-frontend-dev
profiles:
- debug-elasticsearch
depends_on:

22
docker/profiles/docker-compose.gms.yml
View File

@ -178,7 +178,6 @@ services:
#################################
system-update-quickstart:
<<: *datahub-system-update-service
container_name: system-update
profiles:
- quickstart
- quickstart-storage
@ -194,7 +193,6 @@ services:
condition: service_completed_successfully
system-update-quickstart-cassandra:
<<: *datahub-system-update-service
container_name: system-update
profiles:
- quickstart-cassandra
environment:
@ -210,7 +208,6 @@ services:
condition: service_completed_successfully
system-update-quickstart-postgres:
<<: *datahub-system-update-service
container_name: system-update
profiles:
- quickstart-postgres
environment:
@ -224,7 +221,6 @@ services:
condition: service_completed_successfully
system-update-debug:
<<: *datahub-system-update-service-dev
container_name: system-update-dev
profiles:
- debug
- debug-backend
@ -238,7 +234,6 @@ services:
condition: service_completed_successfully
system-update-debug-elasticsearch:
<<: *datahub-system-update-service-dev
container_name: system-update-dev
profiles:
- debug-elasticsearch
depends_on:
@ -250,7 +245,6 @@ services:
condition: service_completed_successfully
system-update-debug-postgres:
<<: *datahub-system-update-service-dev
container_name: system-update-dev
profiles:
- debug-postgres
environment:
@ -264,7 +258,6 @@ services:
condition: service_completed_successfully
system-update-debug-cassandra:
<<: *datahub-system-update-service-dev
container_name: system-update-dev
profiles:
- debug-cassandra
environment:
@ -278,7 +271,6 @@ services:
condition: service_completed_successfully
system-update-debug-neo4j:
<<: *datahub-system-update-service-dev
container_name: system-update-dev
profiles:
- debug-neo4j
environment:
@ -298,7 +290,6 @@ services:
profiles:
- quickstart
- quickstart-backend
container_name: datahub-gms
depends_on:
system-update-quickstart:
condition: service_completed_successfully
@ -306,7 +297,6 @@ services:
<<: *datahub-gms-service
profiles:
- quickstart-cassandra
container_name: datahub-gms
environment:
<<: [*primary-datastore-cassandra-env, *graph-datastore-neo4j-env, *datahub-gms-env]
depends_on:
@ -316,7 +306,6 @@ services:
<<: *datahub-gms-service
profiles:
- quickstart-postgres
container_name: datahub-gms
environment:
<<: [*primary-datastore-postgres-env, *datahub-gms-env]
depends_on:
@ -326,7 +315,6 @@ services:
<<: *datahub-gms-service
profiles:
- quickstart-consumers
container_name: datahub-gms
environment:
<<: *datahub-gms-env
MAE_CONSUMER_ENABLED: false
@ -339,7 +327,6 @@ services:
profiles:
- debug
- debug-backend
container_name: datahub-gms-dev
depends_on:
system-update-debug:
condition: service_completed_successfully
@ -349,7 +336,6 @@ services:
- debug-postgres
environment:
<<: [*primary-datastore-postgres-env, *datahub-gms-dev-env]
container_name: datahub-gms-dev
depends_on:
system-update-debug-postgres:
condition: service_completed_successfully
@ -359,7 +345,6 @@ services:
- debug-cassandra
environment:
<<: [*primary-datastore-cassandra-env, *datahub-gms-dev-env]
container_name: datahub-gms-dev
depends_on:
system-update-debug-cassandra:
condition: service_completed_successfully
@ -371,7 +356,6 @@ services:
<<: *datahub-gms-dev-env
MAE_CONSUMER_ENABLED: false
MCE_CONSUMER_ENABLED: false
container_name: datahub-gms-dev
depends_on:
system-update-debug:
condition: service_completed_successfully
@ -381,7 +365,6 @@ services:
- debug-neo4j
environment:
<<: [*graph-datastore-neo4j-env, *datahub-gms-dev-env]
container_name: datahub-gms-dev
depends_on:
system-update-debug-neo4j:
condition: service_completed_successfully
@ -389,7 +372,6 @@ services:
<<: *datahub-gms-service-dev
profiles:
- debug-elasticsearch
container_name: datahub-gms-dev
depends_on:
system-update-debug-elasticsearch:
condition: service_completed_successfully
@ -400,7 +382,6 @@ services:
<<: *datahub-mae-consumer-service
profiles:
- quickstart-consumers
container_name: datahub-mae-consumer
depends_on:
datahub-gms-quickstart-consumers:
condition: service_healthy
@ -408,7 +389,6 @@ services:
<<: *datahub-mae-consumer-service-dev
profiles:
- debug-consumers
container_name: datahub-mae-consumer-dev
depends_on:
datahub-gms-debug-consumers:
condition: service_healthy
@ -419,7 +399,6 @@ services:
<<: *datahub-mce-consumer-service
profiles:
- quickstart-consumers
container_name: datahub-mce-consumer
depends_on:
datahub-gms-quickstart-consumers:
condition: service_healthy
@ -427,7 +406,6 @@ services:
<<: *datahub-mce-consumer-service-dev
profiles:
- debug-consumers
container_name: datahub-mce-consumer-dev
depends_on:
datahub-gms-debug-consumers:
condition: service_healthy

17
docker/profiles/docker-compose.prerequisites.yml
View File

@ -106,7 +106,6 @@ x-profiles-dev: &profiles-dev
services:
mysql:
container_name: mysql
profiles: *mysql-profiles
hostname: mysql
image: mysql:${DATAHUB_MYSQL_VERSION:-8.2}
@ -125,7 +124,6 @@ services:
- ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
- mysqldata:/var/lib/mysql
mysql-setup: &mysql-setup
container_name: mysql-setup
profiles: *mysql-profiles-quickstart
hostname: mysql-setup
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mysql-setup}:${DATAHUB_VERSION:-head}
@ -137,11 +135,9 @@ services:
datahub_setup_job: true
mysql-setup-dev:
<<: *mysql-setup
container_name: mysql-setup-dev
profiles: *mysql-profiles-dev
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mysql-setup}:debug
postgres:
container_name: postgres
profiles: *postgres-profiles
hostname: postgres
image: postgres:${DATAHUB_POSTGRES_VERSION:-15.5}
@ -159,7 +155,6 @@ services:
- ./postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
- postgresdata:/var/lib/postgresql/data
postgres-setup: &postgres-setup
container_name: postgres-setup
profiles: *postgres-profiles-quickstart
hostname: postgres-setup
image: ${DATAHUB_POSTGRES_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-postgres-setup}:${DATAHUB_VERSION:-head}
@ -171,11 +166,9 @@ services:
datahub_setup_job: true
postgres-setup-dev:
<<: *postgres-setup
container_name: postgres-setup-dev
profiles: *postgres-profiles-dev
image: ${DATAHUB_POSTGRES_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-postgres-setup}:debug
cassandra:
container_name: cassandra
profiles: *cassandra-profiles
hostname: cassandra
image: cassandra:4.1
@ -189,7 +182,6 @@ services:
volumes:
- cassandradata:/var/lib/cassandra
cassandra-setup:
container_name: cassandra-setup
profiles: *cassandra-profiles
hostname: cassandra-setup
image: cassandra:4.1
@ -202,7 +194,6 @@ services:
labels:
datahub_setup_job: true
neo4j:
container_name: neo4j
profiles: *neo4j-profiles
hostname: neo4j
image: neo4j:4.4.28-community
@ -219,7 +210,6 @@ services:
volumes:
- neo4jdata:/data
kafka-broker:
container_name: broker
hostname: broker
image: confluentinc/cp-kafka:7.4.0
command:
@ -264,7 +254,6 @@ services:
volumes:
- broker:/var/lib/kafka/data/
kafka-setup: &kafka-setup
container_name: kafka-setup
profiles: *profiles-quickstart
hostname: kafka-setup
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
@ -280,14 +269,12 @@ services:
datahub_setup_job: true
kafka-setup-dev:
<<: *kafka-setup
container_name: kafka-setup-dev
profiles: *profiles-dev
environment:
<<: *kafka-setup-env
DATAHUB_PRECREATE_TOPICS: ${DATAHUB_PRECREATE_TOPICS:-true}
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-kafka-setup}:debug
elasticsearch:
container_name: elasticsearch
profiles: *elasticsearch-profiles
hostname: search
image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1}
@ -310,7 +297,6 @@ services:
volumes:
- esdata:/usr/share/elasticsearch/data
elasticsearch-setup-dev: &elasticsearch-setup-dev
container_name: elasticsearch-setup-dev
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-elasticsearch-setup}:debug
profiles: *elasticsearch-profiles
hostname: elasticsearch-setup
@ -324,7 +310,6 @@ services:
labels:
datahub_setup_job: true
opensearch:
container_name: opensearch
profiles: *opensearch-profiles
hostname: search
image: ${DATAHUB_SEARCH_IMAGE:-opensearchproject/opensearch}:${DATAHUB_SEARCH_TAG:-2.9.0}
@ -348,7 +333,6 @@ services:
- osdata:/usr/share/elasticsearch/data
opensearch-setup: &opensearch-setup
<<: *elasticsearch-setup-dev
container_name: opensearch-setup
profiles: *opensearch-profiles-quickstart
hostname: opensearch-setup
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
@ -362,7 +346,6 @@ services:
datahub_setup_job: true
opensearch-setup-dev:
<<: *opensearch-setup
container_name: opensearch-setup-dev
profiles: *opensearch-profiles-dev
hostname: opensearch-setup-dev
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-elasticsearch-setup}:debug

13
docker/quickstart/docker-compose-m1.quickstart.yml
View File

@ -3,7 +3,6 @@ networks:
name: datahub_network
services:
broker:
container_name: broker
depends_on:
zookeeper:
condition: service_healthy
@ -31,7 +30,6 @@ services:
volumes:
- broker:/var/lib/kafka/data/
datahub-actions:
container_name: datahub-actions
depends_on:
datahub-gms:
condition: service_healthy
@ -51,7 +49,6 @@ services:
hostname: actions
image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head}
datahub-frontend-react:
container_name: datahub-frontend-react
depends_on:
datahub-gms:
condition: service_healthy
@ -73,7 +70,6 @@ services:
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-gms:
container_name: datahub-gms
depends_on:
datahub-upgrade:
condition: service_completed_successfully
@ -124,7 +120,6 @@ services:
command:
- -u
- SystemUpdate
container_name: datahub-upgrade
depends_on:
elasticsearch-setup:
condition: service_completed_successfully
@ -158,7 +153,6 @@ services:
labels:
datahub_setup_job: true
elasticsearch:
container_name: elasticsearch
deploy:
resources:
limits:
@ -181,7 +175,6 @@ services:
volumes:
- esdata:/usr/share/elasticsearch/data
elasticsearch-setup:
container_name: elasticsearch-setup
depends_on:
elasticsearch:
condition: service_healthy
@ -196,7 +189,6 @@ services:
labels:
datahub_setup_job: true
kafka-setup:
container_name: kafka-setup
depends_on:
broker:
condition: service_healthy
@ -213,7 +205,6 @@ services:
datahub_setup_job: true
mysql:
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
container_name: mysql
environment:
- MYSQL_DATABASE=datahub
- MYSQL_USER=datahub
@ -234,7 +225,6 @@ services:
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
- mysqldata:/var/lib/mysql
mysql-setup:
container_name: mysql-setup
depends_on:
mysql:
condition: service_healthy
@ -249,7 +239,6 @@ services:
labels:
datahub_setup_job: true
neo4j:
container_name: neo4j
environment:
- NEO4J_AUTH=neo4j/datahub
- NEO4J_dbms_default__database=graph.db
@ -269,7 +258,6 @@ services:
volumes:
- neo4jdata:/data
schema-registry:
container_name: schema-registry
depends_on:
broker:
condition: service_healthy
@ -288,7 +276,6 @@ services:
ports:
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
zookeeper:
container_name: zookeeper
environment:
- ZOOKEEPER_CLIENT_PORT=2181
- ZOOKEEPER_TICK_TIME=2000

12
docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
View File

@ -3,7 +3,6 @@ networks:
name: datahub_network
services:
broker:
container_name: broker
depends_on:
zookeeper:
condition: service_healthy
@ -31,7 +30,6 @@ services:
volumes:
- broker:/var/lib/kafka/data/
datahub-actions:
container_name: datahub-actions
depends_on:
datahub-gms:
condition: service_healthy
@ -51,7 +49,6 @@ services:
hostname: actions
image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head}
datahub-frontend-react:
container_name: datahub-frontend-react
depends_on:
datahub-gms:
condition: service_healthy
@ -73,7 +70,6 @@ services:
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-gms:
container_name: datahub-gms
depends_on:
datahub-upgrade:
condition: service_completed_successfully
@ -119,7 +115,6 @@ services:
command:
- -u
- SystemUpdate
container_name: datahub-upgrade
depends_on:
elasticsearch-setup:
condition: service_completed_successfully
@ -151,7 +146,6 @@ services:
labels:
datahub_setup_job: true
elasticsearch:
container_name: elasticsearch
deploy:
resources:
limits:
@ -174,7 +168,6 @@ services:
volumes:
- esdata:/usr/share/elasticsearch/data
elasticsearch-setup:
container_name: elasticsearch-setup
depends_on:
elasticsearch:
condition: service_healthy
@ -189,7 +182,6 @@ services:
labels:
datahub_setup_job: true
kafka-setup:
container_name: kafka-setup
depends_on:
broker:
condition: service_healthy
@ -206,7 +198,6 @@ services:
datahub_setup_job: true
mysql:
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
container_name: mysql
environment:
- MYSQL_DATABASE=datahub
- MYSQL_USER=datahub
@ -227,7 +218,6 @@ services:
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
- mysqldata:/var/lib/mysql
mysql-setup:
container_name: mysql-setup
depends_on:
mysql:
condition: service_healthy
@ -242,7 +232,6 @@ services:
labels:
datahub_setup_job: true
schema-registry:
container_name: schema-registry
depends_on:
broker:
condition: service_healthy
@ -261,7 +250,6 @@ services:
ports:
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
zookeeper:
container_name: zookeeper
environment:
- ZOOKEEPER_CLIENT_PORT=2181
- ZOOKEEPER_TICK_TIME=2000

12
docker/quickstart/docker-compose-without-neo4j.quickstart.yml
View File

@ -3,7 +3,6 @@ networks:
name: datahub_network
services:
broker:
container_name: broker
depends_on:
zookeeper:
condition: service_healthy
@ -31,7 +30,6 @@ services:
volumes:
- broker:/var/lib/kafka/data/
datahub-actions:
container_name: datahub-actions
depends_on:
datahub-gms:
condition: service_healthy
@ -51,7 +49,6 @@ services:
hostname: actions
image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head}
datahub-frontend-react:
container_name: datahub-frontend-react
depends_on:
datahub-gms:
condition: service_healthy
@ -73,7 +70,6 @@ services:
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-gms:
container_name: datahub-gms
depends_on:
datahub-upgrade:
condition: service_completed_successfully
@ -119,7 +115,6 @@ services:
command:
- -u
- SystemUpdate
container_name: datahub-upgrade
depends_on:
elasticsearch-setup:
condition: service_completed_successfully
@ -151,7 +146,6 @@ services:
labels:
datahub_setup_job: true
elasticsearch:
container_name: elasticsearch
deploy:
resources:
limits:
@ -174,7 +168,6 @@ services:
volumes:
- esdata:/usr/share/elasticsearch/data
elasticsearch-setup:
container_name: elasticsearch-setup
depends_on:
elasticsearch:
condition: service_healthy
@ -189,7 +182,6 @@ services:
labels:
datahub_setup_job: true
kafka-setup:
container_name: kafka-setup
depends_on:
broker:
condition: service_healthy
@ -206,7 +198,6 @@ services:
datahub_setup_job: true
mysql:
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
container_name: mysql
environment:
- MYSQL_DATABASE=datahub
- MYSQL_USER=datahub
@ -227,7 +218,6 @@ services:
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
- mysqldata:/var/lib/mysql
mysql-setup:
container_name: mysql-setup
depends_on:
mysql:
condition: service_healthy
@ -242,7 +232,6 @@ services:
labels:
datahub_setup_job: true
schema-registry:
container_name: schema-registry
depends_on:
broker:
condition: service_healthy
@ -261,7 +250,6 @@ services:
ports:
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
zookeeper:
container_name: zookeeper
environment:
- ZOOKEEPER_CLIENT_PORT=2181
- ZOOKEEPER_TICK_TIME=2000

2
docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml
View File

@ -4,7 +4,6 @@ services:
- MAE_CONSUMER_ENABLED=false
- MCE_CONSUMER_ENABLED=false
datahub-mae-consumer:
container_name: datahub-mae-consumer
environment:
- KAFKA_CONSUMER_STOP_ON_DESERIALIZATION_ERROR=${KAFKA_CONSUMER_STOP_ON_DESERIALIZATION_ERROR:-true}
- KAFKA_CONSUMER_HEALTH_CHECK_ENABLED=${KAFKA_CONSUMER_HEALTH_CHECK_ENABLED:-true}
@ -25,7 +24,6 @@ services:
ports:
- 9091:9091
datahub-mce-consumer:
container_name: datahub-mce-consumer
environment:
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_system

2
docker/quickstart/docker-compose.consumers.quickstart.yml
View File

@ -4,7 +4,6 @@ services:
- MAE_CONSUMER_ENABLED=false
- MCE_CONSUMER_ENABLED=false
datahub-mae-consumer:
container_name: datahub-mae-consumer
depends_on:
neo4j:
condition: service_healthy
@ -32,7 +31,6 @@ services:
ports:
- 9091:9091
datahub-mce-consumer:
container_name: datahub-mce-consumer
depends_on:
neo4j:
condition: service_healthy

1
docker/quickstart/docker-compose.monitoring.quickstart.yml
View File

@ -36,7 +36,6 @@ services:
- '14268'
- '14250'
prometheus:
container_name: prometheus
image: prom/prometheus:latest
ports:
- 9089:9090

13
docker/quickstart/docker-compose.quickstart.yml
View File

@ -3,7 +3,6 @@ networks:
name: datahub_network
services:
broker:
container_name: broker
depends_on:
zookeeper:
condition: service_healthy
@ -31,7 +30,6 @@ services:
volumes:
- broker:/var/lib/kafka/data/
datahub-actions:
container_name: datahub-actions
depends_on:
datahub-gms:
condition: service_healthy
@ -51,7 +49,6 @@ services:
hostname: actions
image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head}
datahub-frontend-react:
container_name: datahub-frontend-react
depends_on:
datahub-gms:
condition: service_healthy
@ -73,7 +70,6 @@ services:
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-gms:
container_name: datahub-gms
depends_on:
datahub-upgrade:
condition: service_completed_successfully
@ -124,7 +120,6 @@ services:
command:
- -u
- SystemUpdate
container_name: datahub-upgrade
depends_on:
elasticsearch-setup:
condition: service_completed_successfully
@ -158,7 +153,6 @@ services:
labels:
datahub_setup_job: true
elasticsearch:
container_name: elasticsearch
deploy:
resources:
limits:
@ -181,7 +175,6 @@ services:
volumes:
- esdata:/usr/share/elasticsearch/data
elasticsearch-setup:
container_name: elasticsearch-setup
depends_on:
elasticsearch:
condition: service_healthy
@ -196,7 +189,6 @@ services:
labels:
datahub_setup_job: true
kafka-setup:
container_name: kafka-setup
depends_on:
broker:
condition: service_healthy
@ -213,7 +205,6 @@ services:
datahub_setup_job: true
mysql:
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
container_name: mysql
environment:
- MYSQL_DATABASE=datahub
- MYSQL_USER=datahub
@ -234,7 +225,6 @@ services:
- ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
- mysqldata:/var/lib/mysql
mysql-setup:
container_name: mysql-setup
depends_on:
mysql:
condition: service_healthy
@ -249,7 +239,6 @@ services:
labels:
datahub_setup_job: true
neo4j:
container_name: neo4j
environment:
- NEO4J_AUTH=neo4j/datahub
- NEO4J_dbms_default__database=graph.db
@ -269,7 +258,6 @@ services:
volumes:
- neo4jdata:/data
schema-registry:
container_name: schema-registry
depends_on:
broker:
condition: service_healthy
@ -288,7 +276,6 @@ services:
ports:
- ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
zookeeper:
container_name: zookeeper
environment:
- ZOOKEEPER_CLIENT_PORT=2181
- ZOOKEEPER_TICK_TIME=2000

4
metadata-ingestion/src/datahub/cli/docker_cli.py
View File

@ -281,7 +281,7 @@ def _backup(backup_file: str) -> int:
[
"bash",
"-c",
f"docker exec mysql mysqldump -u root -pdatahub datahub > {resolved_backup_file}",
f"docker exec {DOCKER_COMPOSE_PROJECT_NAME}-mysql-1 mysqldump -u root -pdatahub datahub > {resolved_backup_file}",
]
)
logger.info(
@ -321,7 +321,7 @@ def _restore(
[
"bash",
"-c",
"docker exec -i mysql bash -c 'mysql -uroot -pdatahub datahub '",
f"docker exec -i {DOCKER_COMPOSE_PROJECT_NAME}-mysql-1 bash -c 'mysql -uroot -pdatahub datahub '",
],
stdin=fp,
stdout=subprocess.PIPE,

2
smoke-test/tests/consistency_utils.py
View File

@ -22,7 +22,7 @@ def wait_for_writes_to_sync(max_timeout_in_sec: int = 120) -> None:
while not lag_zero and (time.time() - start_time) < max_timeout_in_sec:
time.sleep(1) # micro-sleep
completed_process = subprocess.run(
"docker exec broker /bin/kafka-consumer-groups --bootstrap-server broker:29092 --group generic-mae-consumer-job-client --describe | grep -v LAG | awk '{print $6}'",
"docker exec datahub-broker-1 /bin/kafka-consumer-groups --bootstrap-server broker:29092 --group generic-mae-consumer-job-client --describe | grep -v LAG | awk '{print $6}'",
capture_output=True,
shell=True,
text=True,