yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-28 02:17:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(docker): docker compose profiles updates (#9514)

Browse Source 
Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
This commit is contained in:
david-leifker 2024-01-03 15:58:50 -06:00 committed by GitHub
parent 822d0eb014
commit 296e41dfed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 30 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docker/build.gradle
View File

@ -31,6 +31,11 @@ ext {
pg_quickstart_modules = quickstart_modules - [':docker:mysql-setup'] + [':docker:postgres-setup']
}
tasks.register('minDockerCompose2.20', Exec) {
executable 'bash'
args '-c', 'echo -e "$(docker compose version --short)\n2.20"|sort --version-sort --check=quiet --reverse'
}
tasks.register('quickstart') {}
tasks.register('quickstartSlim') {}
tasks.register('quickstartDebug') {}
@ -118,9 +123,9 @@ tasks.getByName('quickstartDebugComposeUp').dependsOn(
)
tasks.withType(ComposeUp).configureEach {
shouldRunAfter('quickstartNuke')
dependsOn tasks.named("minDockerCompose2.20")
}
task debugReload(type: Exec) {
def cmd = ['docker compose -p datahub --profile debug'] + compose_args + ['restart'] + debug_reloadable
commandLine 'bash', '-c', cmd.join(" ")

2
docker/profiles/README.md
View File

@ -5,7 +5,7 @@ for quickstart use-cases as well as development use-cases. These configurations
infrastructure configurations that DataHub can operate on.
Requirements:
* Use the profiles requires a modern version of docker.
* Using profiles requires docker compose >= 2.20.
* If using the debug/development profiles, you will need to have built the `debug` docker images locally. See the Development Profiles section for more details.
```bash

2
docker/profiles/docker-compose.actions.yml
View File

@ -1,7 +1,7 @@
x-datahub-actions-service: &datahub-actions-service
hostname: actions
image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head}
image: ${DATAHUB_ACTIONS_IMAGE:-${DATAHUB_ACTIONS_REPO:-acryldata}/datahub-actions}:${ACTIONS_VERSION:-head}
env_file: datahub-actions/env/docker.env
environment:
ACTIONS_EXTRA_PACKAGES: ${ACTIONS_EXTRA_PACKAGES:-}

4
docker/profiles/docker-compose.frontend.yml
View File

@ -1,7 +1,7 @@
x-datahub-frontend-service: &datahub-frontend-service
hostname: datahub-frontend-react
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
image: ${DATAHUB_FRONTEND_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-frontend-react}:${DATAHUB_VERSION:-head}
ports:
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
env_file: datahub-frontend/env/docker.env
@ -12,7 +12,7 @@ x-datahub-frontend-service: &datahub-frontend-service
x-datahub-frontend-service-dev: &datahub-frontend-service-dev
<<: *datahub-frontend-service
image: linkedin/datahub-frontend-react:debug
image: ${DATAHUB_FRONTEND_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-frontend-react}:debug
ports:
- ${DATAHUB_MAPPED_FRONTEND_DEBUG_PORT:-5002}:5002
- ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002

16
docker/profiles/docker-compose.gms.yml
View File

@ -54,7 +54,7 @@ x-datahub-dev-telemetry-env: &datahub-dev-telemetry-env
#################################
x-datahub-system-update-service: &datahub-system-update-service
hostname: datahub-system-update
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}
image: ${DATAHUB_UPGRADE_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-upgrade}:${DATAHUB_VERSION:-head}
command:
- -u
- SystemUpdate
@ -67,7 +67,7 @@ x-datahub-system-update-service: &datahub-system-update-service
x-datahub-system-update-service-dev: &datahub-system-update-service-dev
<<: *datahub-system-update-service
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:debug
image: ${DATAHUB_UPGRADE_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-upgrade}:debug
ports:
- ${DATAHUB_MAPPED_UPGRADE_DEBUG_PORT:-5003}:5003
environment: &datahub-system-update-dev-env
@ -85,7 +85,7 @@ x-datahub-system-update-service-dev: &datahub-system-update-service-dev
#################################
x-datahub-gms-service: &datahub-gms-service
hostname: datahub-gms
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
image: ${DATAHUB_GMS_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-gms}:${DATAHUB_VERSION:-head}
ports:
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
env_file: datahub-gms/env/docker.env
@ -102,7 +102,7 @@ x-datahub-gms-service: &datahub-gms-service
x-datahub-gms-service-dev: &datahub-gms-service-dev
<<: *datahub-gms-service
image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:debug
image: ${DATAHUB_GMS_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-gms}:debug
ports:
- ${DATAHUB_MAPPED_GMS_DEBUG_PORT:-5001}:5001
- ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
@ -128,7 +128,7 @@ x-datahub-gms-service-dev: &datahub-gms-service-dev
#################################
x-datahub-mae-consumer-service: &datahub-mae-consumer-service
hostname: datahub-mae-consumer
image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:${DATAHUB_VERSION:-head}
image: ${DATAHUB_MAE_CONSUMER_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-mae-consumer}:${DATAHUB_VERSION:-head}
ports:
- 9091:9091
env_file: datahub-mae-consumer/env/docker.env
@ -137,7 +137,7 @@ x-datahub-mae-consumer-service: &datahub-mae-consumer-service
x-datahub-mae-consumer-service-dev: &datahub-mae-consumer-service-dev
<<: *datahub-mae-consumer-service
image: ${DATAHUB_MAE_CONSUMER_IMAGE:-linkedin/datahub-mae-consumer}:debug
image: ${DATAHUB_MAE_CONSUMER_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-mae-consumer}:debug
environment:
<<: [*datahub-dev-telemetry-env, *datahub-mae-consumer-env]
volumes:
@ -151,7 +151,7 @@ x-datahub-mae-consumer-service-dev: &datahub-mae-consumer-service-dev
#################################
x-datahub-mce-consumer-service: &datahub-mce-consumer-service
hostname: datahub-mce-consumer
image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:${DATAHUB_VERSION:-head}
image: ${DATAHUB_MCE_CONSUMER_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-mce-consumer}:${DATAHUB_VERSION:-head}
ports:
- 9090:9090
env_file: datahub-mce-consumer/env/docker.env
@ -160,7 +160,7 @@ x-datahub-mce-consumer-service: &datahub-mce-consumer-service
x-datahub-mce-consumer-service-dev: &datahub-mce-consumer-service-dev
<<: *datahub-mce-consumer-service
image: ${DATAHUB_MCE_CONSUMER_IMAGE:-linkedin/datahub-mce-consumer}:debug
image: ${DATAHUB_MCE_CONSUMER_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-mce-consumer}:debug
environment:
<<: [*datahub-dev-telemetry-env, *datahub-mce-consumer-env]
volumes:

18
docker/profiles/docker-compose.prerequisites.yml
View File

@ -128,7 +128,7 @@ services:
container_name: mysql-setup
profiles: *mysql-profiles-quickstart
hostname: mysql-setup
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head}
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mysql-setup}:${DATAHUB_VERSION:-head}
env_file: mysql-setup/env/docker.env
depends_on:
mysql:
@ -139,7 +139,7 @@ services:
<<: *mysql-setup
container_name: mysql-setup-dev
profiles: *mysql-profiles-dev
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:debug
image: ${DATAHUB_MYSQL_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mysql-setup}:debug
postgres:
container_name: postgres
profiles: *postgres-profiles
@ -162,7 +162,7 @@ services:
container_name: postgres-setup
profiles: *postgres-profiles-quickstart
hostname: postgres-setup
image: ${DATAHUB_POSTGRES_SETUP_IMAGE:-acryldata/datahub-postgres-setup}:${DATAHUB_VERSION:-head}
image: ${DATAHUB_POSTGRES_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-postgres-setup}:${DATAHUB_VERSION:-head}
env_file: postgres-setup/env/docker.env
depends_on:
postgres:
@ -173,7 +173,7 @@ services:
<<: *postgres-setup
container_name: postgres-setup-dev
profiles: *postgres-profiles-dev
image: ${DATAHUB_POSTGRES_SETUP_IMAGE:-acryldata/datahub-postgres-setup}:debug
image: ${DATAHUB_POSTGRES_SETUP_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-postgres-setup}:debug
cassandra:
container_name: cassandra
profiles: *cassandra-profiles
@ -267,7 +267,7 @@ services:
container_name: kafka-setup
profiles: *profiles-quickstart
hostname: kafka-setup
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-kafka-setup}:${DATAHUB_VERSION:-head}
env_file: kafka-setup/env/docker.env
environment: &kafka-setup-env
DATAHUB_PRECREATE_TOPICS: ${DATAHUB_PRECREATE_TOPICS:-false}
@ -285,7 +285,7 @@ services:
environment:
<<: *kafka-setup-env
DATAHUB_PRECREATE_TOPICS: ${DATAHUB_PRECREATE_TOPICS:-true}
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:debug
image: ${DATAHUB_KAFKA_SETUP_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-kafka-setup}:debug
elasticsearch:
container_name: elasticsearch
profiles: *elasticsearch-profiles
@ -311,7 +311,7 @@ services:
- esdata:/usr/share/elasticsearch/data
elasticsearch-setup-dev: &elasticsearch-setup-dev
container_name: elasticsearch-setup-dev
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:debug
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-elasticsearch-setup}:debug
profiles: *elasticsearch-profiles
hostname: elasticsearch-setup
env_file: elasticsearch-setup/env/docker.env
@ -351,7 +351,7 @@ services:
container_name: opensearch-setup
profiles: *opensearch-profiles-quickstart
hostname: opensearch-setup
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head}
environment:
<<: *search-datastore-environment
USE_AWS_ELASTICSEARCH: ${USE_AWS_ELASTICSEARCH:-true}
@ -365,7 +365,7 @@ services:
container_name: opensearch-setup-dev
profiles: *opensearch-profiles-dev
hostname: opensearch-setup-dev
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:debug
image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-${DATAHUB_REPO:-linkedin}/datahub-elasticsearch-setup}:debug
environment:
<<: *search-datastore-environment
USE_AWS_ELASTICSEARCH: ${USE_AWS_ELASTICSEARCH:-true}

2
docs/developers.md
View File

@ -9,7 +9,7 @@ title: "Local Development"
- [Java 17 JDK](https://openjdk.org/projects/jdk/17/)
- [Python 3.10](https://www.python.org/downloads/release/python-3100/)
- [Docker](https://www.docker.com/)
- [Docker Compose](https://docs.docker.com/compose/)
- [Docker Compose >=2.20](https://docs.docker.com/compose/)
- Docker engine with at least 8GB of memory to run tests.
:::

3
docs/how/updating-datahub.md
View File

@ -8,7 +8,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
- Updating MySQL version for quickstarts to 8.2, may cause quickstart issues for existing instances.
- Neo4j 5.x, may require migration from 4.x
- Build now requires JDK17 (Runtime Java 11)
- Build requires JDK17 (Runtime Java 11)
- Build requires Docker Compose > 2.20
### Potential Downtime