yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-06-27 05:03:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(docker-compose): add localstack to compose profiles (#13650)

Browse Source
This commit is contained in:
david-leifker 2025-05-28 15:11:27 -05:00 committed by GitHub
parent 8787892cbb
commit 7cb26edb5b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 49 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docker/build.gradle
View File

@ -98,6 +98,14 @@ ext {
profile: 'quickstart-storage',
preserveVolumes: true
],
'quickstartBackendDebug': [
profile: 'debug-backend-aws',
modules: python_services_modules + backend_profile_modules + [':datahub-frontend', ':datahub-actions'],
isDebug: true,
additionalEnv: [
DATAHUB_LOCAL_ACTIONS_ENV: "${rootProject.project(':smoke-test').projectDir}/test_resources/actions/actions.env"
]
],
'allImages': [ //This is a special task just to include all images as dependencies - and is useful when CI needs to publish all images
profile: 'quickstart-consumers',
modules: python_services_modules + backend_profile_modules + [

21
docker/profiles/README.md
View File

@ -95,16 +95,17 @@ Runs everything except for the GMS. Useful for running just a local (non-docker)
### Development Profiles Table
| Profile Name | MySQL | Postgres | Cassandra | Neo4j | Frontend | GMS | Actions | SystemUpdate | MAE | MCE | Kafka | OpenSearch | Elasticsearch |
| ------------------- | ----- | -------- | --------- | ----- | -------- | --- | ------- | ------------ | --- | --- | ----- | ---------- | ------------- |
| debug | X | | | | X | X | X | X | | | X | X | |
| debug-frontend | X | | | | X | | | X | | | X | X | |
| debug-backend | X | | | | | X | X | X | | | X | X | |
| debug-postgres | | X | | | X | X | X | X | | | X | X | |
| debug-cassandra | | | X | | X | X | X | X | | | X | X | |
| debug-consumers | X | | | | X | X | X | X | X | X | X | X | |
| debug-neo4j | X | | | X | X | X | X | X | | | X | X | |
| debug-elasticsearch | X | | | | X | X | X | X | | | X | | X |
| Profile Name | MySQL | Postgres | Cassandra | Neo4j | Frontend | GMS | Actions | SystemUpdate | MAE | MCE | Kafka | OpenSearch | Elasticsearch | Localstack (AWS) |
| ------------------- | ----- | -------- | --------- | ----- | -------- | --- | ------- | ------------ | --- | --- | ----- | ---------- | ------------- | ---------------- |
| debug | X | | | | X | X | X | X | | | X | X | | |
| debug-frontend | X | | | | X | | | X | | | X | X | | |
| debug-backend | X | | | | | X | X | X | | | X | X | | |
| debug-postgres | | X | | | X | X | X | X | | | X | X | | |
| debug-cassandra | | | X | | X | X | X | X | | | X | X | | |
| debug-consumers | X | | | | X | X | X | X | X | X | X | X | | |
| debug-neo4j | X | | | X | X | X | X | X | | | X | X | | |
| debug-elasticsearch | X | | | | X | X | X | X | | | X | | X | |
| debug-backend-aws | X | | | | | X | X | X | | | X | X | | X |
## Advanced Setups

1
docker/profiles/docker-compose.actions.yml
View File

@ -60,6 +60,7 @@ services:
profiles:
- debug
- debug-backend
- debug-backend-aws
depends_on:
datahub-gms-debug:
condition: service_healthy

2
docker/profiles/docker-compose.gms.yml
View File

@ -254,6 +254,7 @@ services:
- debug
- debug-min
- debug-backend
- debug-backend-aws
depends_on:
mysql-setup-dev:
condition: service_completed_successfully
@ -360,6 +361,7 @@ services:
profiles:
- debug
- debug-backend
- debug-backend-aws
depends_on:
system-update-debug:
condition: service_completed_successfully

27
docker/profiles/docker-compose.prerequisites.yml
View File

@ -20,6 +20,7 @@ x-mysql-profiles-dev: &mysql-profiles-dev
- debug-consumers
- debug-neo4j
- debug-elasticsearch
- debug-backend-aws
x-mysql-profiles: &mysql-profiles
- quickstart
- quickstart-backend
@ -34,6 +35,7 @@ x-mysql-profiles: &mysql-profiles
- debug-consumers
- debug-neo4j
- debug-elasticsearch
- debug-backend-aws
x-postgres-profiles-quickstart: &postgres-profiles-quickstart
- quickstart-postgres
@ -75,6 +77,7 @@ x-opensearch-profiles-dev: &opensearch-profiles-dev
- debug-cassandra
- debug-consumers
- debug-neo4j
- debug-backend-aws
x-opensearch-profiles: &opensearch-profiles
- quickstart
- quickstart-backend
@ -93,6 +96,11 @@ x-opensearch-profiles: &opensearch-profiles
- debug-cassandra
- debug-consumers
- debug-neo4j
- debug-backend-aws
# AWS Infra profiles
x-profiles-aws: &aws-profiles
- debug-backend-aws
# Debug vs Quickstart Profiles
x-profiles-quickstart: &profiles-quickstart
@ -115,6 +123,7 @@ x-profiles-dev: &profiles-dev
- debug-consumers
- debug-neo4j
- debug-elasticsearch
- debug-backend-aws
services:
mysql:
@ -366,6 +375,23 @@ services:
depends_on:
opensearch:
condition: service_healthy
aws-localstack:
profiles: *aws-profiles
hostname: localstack
image: ${LOCALSTACK_IMAGE:-localstack/localstack:4}
ports:
- ${DATAHUB_MAPPED_LOCALSTACK_PORT:-4566}:4566 # LocalStack's main service port
environment:
- AWS_DEFAULT_REGION=${LOCALSTACK_REGION:-us-east-1}
- SERVICES=${LOCALSTACK_SERVICES:-s3,sqs,iam,sts}
volumes:
- localstackdata:/var/lib/localstack
healthcheck:
test: ["CMD", "awslocal", "s3", "ls"]
interval: 5s
timeout: 10s
retries: 3
start_period: 30s
networks:
default:
@ -379,3 +405,4 @@ volumes:
mysqldata:
cassandradata:
postgresdata:
localstackdata: