feat(docker-profiles): version mixing & docs (#12342)

docker/build.gradle

@@ -42,7 +42,6 @@ ext {
                     modules: python_services_modules + backend_profile_modules + [':datahub-frontend'],
                     isDebug: true
             ],
-
             'quickstartDebugConsumers': [
                     profile: 'debug-consumers',
                     modules: python_services_modules + backend_profile_modules + [':datahub-frontend',
@@ -50,7 +49,6 @@ ext {
                                                                           ':metadata-jobs:mae-consumer-job'],
                     isDebug: true
             ],
-
             'quickstartPg': [
                     profile: 'quickstart-postgres',
                     modules: (backend_profile_modules - [':docker:mysql-setup']) + [
@@ -108,9 +106,7 @@ dockerCompose {
             }
 
             // Common environment variables
-            environment.put 'DATAHUB_VERSION', config.isDebug ?
+            environment.put 'DATAHUB_VERSION', System.getenv("DATAHUB_VERSION") ?: "v${version}"
-                    System.getenv("DATAHUB_VERSION") ?: "v${version}" :
-                    "v${version}"
             environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false'
             environment.put "METADATA_TESTS_ENABLED", "true"
             environment.put "DATAHUB_REPO", "${docker_registry}"

docker/profiles/README.md

@@ -101,4 +101,30 @@ Runs everything except for the GMS. Useful for running just a local (non-docker)
 | debug-cassandra     |       |          | X         |       | X        | X   | X       | X            |     |     | X     | X          |               |
 | debug-consumers     | X     |          |           |       | X        | X   | X       | X            | X   | X   | X     | X          |               |
 | debug-neo4j         | X     |          |           | X     | X        | X   | X       | X            |     |     | X     | X          |               |
-| debug-elasticsearch | X     |          |           |       | X        | X   | X       | X            |     |     | X     |            | X             |
+| debug-elasticsearch | X     |          |           |       | X        | X   | X       | X            |     |     | X     |            | X             |
+
+## Advanced Setups
+
+### Version Mixing
+
+In some cases, it might be useful to debug upgrade scenarios where there are intentional version miss-matches. It is possible
+to override individual component versions.
+
+Note: This only works for `non-debug` profiles because of the file mounts when in `debug` which would run older containers
+but still pickup the latest application jars.
+
+In this example we are interested in upgrading two components (the `mae-consumer` and the `mce-consumer`) to a fresh build `v0.15.1-SNAPSHOT`
+while maintaining older components on `v0.14.1` (especially the `system-update` container).
+
+This configuration reproduces the situation where the consumers were upgraded prior to running the latest version of `system-update`. In this
+scenario we expect the consumers to block their startup waiting for the successful completion of a newer `system-update`.
+
+`DATAHUB_VERSION` - specifies the default component version of `v0.14.1`
+`DATAHUB_MAE_VERSION` - specifies an override of just the `mae-consumer` to version `v0.15.1-SNAPSHOT`[1]
+`DATAHUB_MCE_VERSION` - specifies an override of just the `mce-consumer` to version `v0.15.1-SNAPSHOT`[1]
+
+```shell
+ DATAHUB_MAE_VERSION="v0.15.1-SNAPSHOT" DATAHUB_MCE_VERSION="v0.15.1-SNAPSHOT" DATAHUB_VERSION="v0.14.1" ./gradlew quickstart
+```
+
+[1] Image versions were `v0.15.1-SNAPSHOT` built locally prior to running the command.

docker/profiles/docker-compose.gms.yml

@@ -54,7 +54,7 @@ x-datahub-dev-telemetry-env: &datahub-dev-telemetry-env
 #################################
 x-datahub-system-update-service: &datahub-system-update-service
   hostname: datahub-system-update
-  image: ${DATAHUB_UPGRADE_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-upgrade}:${DATAHUB_VERSION:-head}
+  image: ${DATAHUB_UPGRADE_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-upgrade}:${DATAHUB_UPDATE_VERSION:-${DATAHUB_VERSION:-head}}
   command:
     - -u
     - SystemUpdate
@@ -73,7 +73,7 @@ x-datahub-system-update-service: &datahub-system-update-service
 
 x-datahub-system-update-service-dev: &datahub-system-update-service-dev
   <<: *datahub-system-update-service
-  image: ${DATAHUB_UPGRADE_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-upgrade}:${DATAHUB_VERSION:-debug}
+  image: ${DATAHUB_UPGRADE_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-upgrade}:${DATAHUB_UPDATE_VERSION:-${DATAHUB_VERSION:-debug}}
   ports:
     - ${DATAHUB_MAPPED_UPGRADE_DEBUG_PORT:-5003}:5003
   environment: &datahub-system-update-dev-env
@@ -92,7 +92,7 @@ x-datahub-system-update-service-dev: &datahub-system-update-service-dev
 #################################
 x-datahub-gms-service: &datahub-gms-service
   hostname: datahub-gms
-  image: ${DATAHUB_GMS_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-gms}:${DATAHUB_VERSION:-head}
+  image: ${DATAHUB_GMS_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-gms}:${DATAHUB_GMS_VERSION:-${DATAHUB_VERSION:-head}}
   ports:
     - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
   env_file:
@@ -118,7 +118,7 @@ x-datahub-gms-service: &datahub-gms-service
 
 x-datahub-gms-service-dev: &datahub-gms-service-dev
   <<: *datahub-gms-service
-  image: ${DATAHUB_GMS_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-gms}:${DATAHUB_VERSION:-debug}
+  image: ${DATAHUB_GMS_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-gms}:${DATAHUB_GMS_VERSION:-${DATAHUB_VERSION:-debug}}
   ports:
     - ${DATAHUB_MAPPED_GMS_DEBUG_PORT:-5001}:5001
     - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
@@ -150,7 +150,7 @@ x-datahub-gms-service-dev: &datahub-gms-service-dev
 #################################
 x-datahub-mae-consumer-service: &datahub-mae-consumer-service
   hostname: datahub-mae-consumer
-  image: ${DATAHUB_MAE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mae-consumer}:${DATAHUB_VERSION:-head}
+  image: ${DATAHUB_MAE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mae-consumer}:${DATAHUB_MAE_VERSION:-${DATAHUB_VERSION:-head}}
   ports:
     - 9091:9091
   env_file:
@@ -163,7 +163,7 @@ x-datahub-mae-consumer-service: &datahub-mae-consumer-service
 
 x-datahub-mae-consumer-service-dev: &datahub-mae-consumer-service-dev
   <<: *datahub-mae-consumer-service
-  image: ${DATAHUB_MAE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mae-consumer}:${DATAHUB_VERSION:-debug}
+  image: ${DATAHUB_MAE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mae-consumer}:${DATAHUB_MAE_VERSION:-${DATAHUB_VERSION:-debug}}
   environment:
     <<: [*datahub-dev-telemetry-env, *datahub-mae-consumer-env]
     ENTITY_VERSIONING_ENABLED: ${ENTITY_VERSIONING_ENABLED:-true}
@@ -178,7 +178,7 @@ x-datahub-mae-consumer-service-dev: &datahub-mae-consumer-service-dev
 #################################
 x-datahub-mce-consumer-service: &datahub-mce-consumer-service
   hostname: datahub-mce-consumer
-  image: ${DATAHUB_MCE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mce-consumer}:${DATAHUB_VERSION:-head}
+  image: ${DATAHUB_MCE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mce-consumer}:${DATAHUB_MCE_VERSION:-${DATAHUB_VERSION:-head}}
   ports:
     - 9090:9090
   env_file:
@@ -193,7 +193,7 @@ x-datahub-mce-consumer-service: &datahub-mce-consumer-service
 
 x-datahub-mce-consumer-service-dev: &datahub-mce-consumer-service-dev
   <<: *datahub-mce-consumer-service
-  image: ${DATAHUB_MCE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mce-consumer}:${DATAHUB_VERSION:-debug}
+  image: ${DATAHUB_MCE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mce-consumer}:${DATAHUB_MCE_VERSION:-${DATAHUB_VERSION:-debug}}
   environment:
     <<: [*datahub-dev-telemetry-env, *datahub-mce-consumer-env]
     ENTITY_VERSIONING_ENABLED: ${ENTITY_VERSIONING_ENABLED:-true}