From 7cb26edb5bbd2ef3778f37c181c7d31e8d5a1792 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 28 May 2025 15:11:27 -0500 Subject: [PATCH] feat(docker-compose): add localstack to compose profiles (#13650) --- docker/build.gradle | 8 ++++++ docker/profiles/README.md | 21 ++++++++------- docker/profiles/docker-compose.actions.yml | 1 + docker/profiles/docker-compose.gms.yml | 2 ++ .../profiles/docker-compose.prerequisites.yml | 27 +++++++++++++++++++ 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/docker/build.gradle b/docker/build.gradle index 61f957d797..cfa482ec9e 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -98,6 +98,14 @@ ext { profile: 'quickstart-storage', preserveVolumes: true ], + 'quickstartBackendDebug': [ + profile: 'debug-backend-aws', + modules: python_services_modules + backend_profile_modules + [':datahub-frontend', ':datahub-actions'], + isDebug: true, + additionalEnv: [ + DATAHUB_LOCAL_ACTIONS_ENV: "${rootProject.project(':smoke-test').projectDir}/test_resources/actions/actions.env" + ] + ], 'allImages': [ //This is a special task just to include all images as dependencies - and is useful when CI needs to publish all images profile: 'quickstart-consumers', modules: python_services_modules + backend_profile_modules + [ diff --git a/docker/profiles/README.md b/docker/profiles/README.md index 3ac6295fb0..14ff8b8af5 100644 --- a/docker/profiles/README.md +++ b/docker/profiles/README.md @@ -95,16 +95,17 @@ Runs everything except for the GMS. Useful for running just a local (non-docker) ### Development Profiles Table -| Profile Name | MySQL | Postgres | Cassandra | Neo4j | Frontend | GMS | Actions | SystemUpdate | MAE | MCE | Kafka | OpenSearch | Elasticsearch | -| ------------------- | ----- | -------- | --------- | ----- | -------- | --- | ------- | ------------ | --- | --- | ----- | ---------- | ------------- | -| debug | X | | | | X | X | X | X | | | X | X | | -| debug-frontend | X | | | | X | | | X | | | X | X | | -| debug-backend | X | | | | | X | X | X | | | X | X | | -| debug-postgres | | X | | | X | X | X | X | | | X | X | | -| debug-cassandra | | | X | | X | X | X | X | | | X | X | | -| debug-consumers | X | | | | X | X | X | X | X | X | X | X | | -| debug-neo4j | X | | | X | X | X | X | X | | | X | X | | -| debug-elasticsearch | X | | | | X | X | X | X | | | X | | X | +| Profile Name | MySQL | Postgres | Cassandra | Neo4j | Frontend | GMS | Actions | SystemUpdate | MAE | MCE | Kafka | OpenSearch | Elasticsearch | Localstack (AWS) | +| ------------------- | ----- | -------- | --------- | ----- | -------- | --- | ------- | ------------ | --- | --- | ----- | ---------- | ------------- | ---------------- | +| debug | X | | | | X | X | X | X | | | X | X | | | +| debug-frontend | X | | | | X | | | X | | | X | X | | | +| debug-backend | X | | | | | X | X | X | | | X | X | | | +| debug-postgres | | X | | | X | X | X | X | | | X | X | | | +| debug-cassandra | | | X | | X | X | X | X | | | X | X | | | +| debug-consumers | X | | | | X | X | X | X | X | X | X | X | | | +| debug-neo4j | X | | | X | X | X | X | X | | | X | X | | | +| debug-elasticsearch | X | | | | X | X | X | X | | | X | | X | | +| debug-backend-aws | X | | | | | X | X | X | | | X | X | | X | ## Advanced Setups diff --git a/docker/profiles/docker-compose.actions.yml b/docker/profiles/docker-compose.actions.yml index 25ead62090..54119f65d5 100644 --- a/docker/profiles/docker-compose.actions.yml +++ b/docker/profiles/docker-compose.actions.yml @@ -60,6 +60,7 @@ services: profiles: - debug - debug-backend + - debug-backend-aws depends_on: datahub-gms-debug: condition: service_healthy diff --git a/docker/profiles/docker-compose.gms.yml b/docker/profiles/docker-compose.gms.yml index 7f486db436..f7192f8d7f 100644 --- a/docker/profiles/docker-compose.gms.yml +++ b/docker/profiles/docker-compose.gms.yml @@ -254,6 +254,7 @@ services: - debug - debug-min - debug-backend + - debug-backend-aws depends_on: mysql-setup-dev: condition: service_completed_successfully @@ -360,6 +361,7 @@ services: profiles: - debug - debug-backend + - debug-backend-aws depends_on: system-update-debug: condition: service_completed_successfully diff --git a/docker/profiles/docker-compose.prerequisites.yml b/docker/profiles/docker-compose.prerequisites.yml index 9af762fe89..9129aa126d 100644 --- a/docker/profiles/docker-compose.prerequisites.yml +++ b/docker/profiles/docker-compose.prerequisites.yml @@ -20,6 +20,7 @@ x-mysql-profiles-dev: &mysql-profiles-dev - debug-consumers - debug-neo4j - debug-elasticsearch + - debug-backend-aws x-mysql-profiles: &mysql-profiles - quickstart - quickstart-backend @@ -34,6 +35,7 @@ x-mysql-profiles: &mysql-profiles - debug-consumers - debug-neo4j - debug-elasticsearch + - debug-backend-aws x-postgres-profiles-quickstart: &postgres-profiles-quickstart - quickstart-postgres @@ -75,6 +77,7 @@ x-opensearch-profiles-dev: &opensearch-profiles-dev - debug-cassandra - debug-consumers - debug-neo4j + - debug-backend-aws x-opensearch-profiles: &opensearch-profiles - quickstart - quickstart-backend @@ -93,6 +96,11 @@ x-opensearch-profiles: &opensearch-profiles - debug-cassandra - debug-consumers - debug-neo4j + - debug-backend-aws + +# AWS Infra profiles +x-profiles-aws: &aws-profiles + - debug-backend-aws # Debug vs Quickstart Profiles x-profiles-quickstart: &profiles-quickstart @@ -115,6 +123,7 @@ x-profiles-dev: &profiles-dev - debug-consumers - debug-neo4j - debug-elasticsearch + - debug-backend-aws services: mysql: @@ -366,6 +375,23 @@ services: depends_on: opensearch: condition: service_healthy + aws-localstack: + profiles: *aws-profiles + hostname: localstack + image: ${LOCALSTACK_IMAGE:-localstack/localstack:4} + ports: + - ${DATAHUB_MAPPED_LOCALSTACK_PORT:-4566}:4566 # LocalStack's main service port + environment: + - AWS_DEFAULT_REGION=${LOCALSTACK_REGION:-us-east-1} + - SERVICES=${LOCALSTACK_SERVICES:-s3,sqs,iam,sts} + volumes: + - localstackdata:/var/lib/localstack + healthcheck: + test: ["CMD", "awslocal", "s3", "ls"] + interval: 5s + timeout: 10s + retries: 3 + start_period: 30s networks: default: @@ -379,3 +405,4 @@ volumes: mysqldata: cassandradata: postgresdata: + localstackdata: