build: support reload of some modules with env changes (#13325)

docker/build.gradle

@@ -44,6 +44,11 @@ ext {
                     modules: python_services_modules + backend_profile_modules + [':datahub-frontend', ':datahub-actions'],
                     isDebug: true
             ],
+            'quickstartDebugMin': [
+                    profile: 'debug-min',
+                    modules: backend_profile_modules + [':datahub-frontend'],
+                    isDebug: true
+            ],
             'quickstartDebugConsumers': [
                     profile: 'debug-consumers',
                     modules: python_services_modules + backend_profile_modules + [':datahub-frontend',
@@ -98,13 +103,12 @@ ext {
 
     // only for debug variants of quickstart to enable <variant>Reload tasks. 
     // The actual service name needs the profile to be appended,  <container-name>-<profile>
+    // This list only contains modules that can be reloaded via the reloadTask. If other modules need to be reloaded, quickstart* needs to be used.
     moduleToContainer = [
         ':metadata-service:war': 'datahub-gms',
         ':datahub-frontend': 'frontend',
-        ':datahub-upgrade': 'system-update',
         ':metadata-jobs:mce-consumer-job': 'datahub-mce-consumer',
         ':metadata-jobs:mae-consumer-job': 'datahub-mae-consumer',
-        ':datahub-actions': 'datahub-actions',
     ]
 }
 
@@ -301,5 +305,22 @@ quickstart_configs.each { taskName, config ->
                 }    
             }
         }
+        tasks.register("${reloadTaskName}ReloadEnv", Exec) {
+            dependsOn tasks.named("prepareAll${taskName}")
+            group = 'quickstart'
+            description = "Build changed containers but recreate all services for the ${taskName} task"
+            doFirst {
+                def containersToRestart = []
+                moduleToContainer.each { modulePath, containerName ->
+                    // Find which of of the reloadable modules are in used in this task
+                    if (config.modules.contains(modulePath)) {
+                        containersToRestart << "${containerName}-${config.profile}"
+                    }
+                }
+                def cmd = ["docker compose -p datahub --profile ${config.profile}"] + ['-f', compose_base] + ['up', '-d', '--no-deps'] + containersToRestart
+                println(cmd.join(" "))
+                commandLine 'bash', '-c', cmd.join(" ")
+            }
+        }
     }
 }

docker/profiles/docker-compose.frontend.yml

@@ -66,6 +66,13 @@ services:
     depends_on:
       system-update-debug:
         condition: service_completed_successfully
+  frontend-debug-min:
+    <<: *datahub-frontend-service-dev
+    profiles:
+      - debug-min
+    depends_on:
+      system-update-debug:
+        condition: service_completed_successfully
   frontend-debug-frontend:
     <<: *datahub-frontend-service-dev
     profiles:

docker/profiles/docker-compose.gms.yml

@@ -252,6 +252,7 @@ services:
     <<: *datahub-system-update-service-dev
     profiles:
       - debug
+      - debug-min
       - debug-backend
     depends_on:
       mysql-setup-dev:
@@ -362,6 +363,14 @@ services:
     depends_on:
       system-update-debug:
         condition: service_completed_successfully
+  datahub-gms-debug-min:
+    <<: *datahub-gms-service-dev
+    profiles:
+      - debug-min
+      - debug-backend
+    depends_on:
+      system-update-debug:
+        condition: service_completed_successfully
   datahub-gms-debug-postgres:
     <<: *datahub-gms-service-dev
     profiles:

docker/profiles/docker-compose.prerequisites.yml

@@ -14,6 +14,7 @@ x-mysql-profiles-quickstart: &mysql-profiles-quickstart
   - quickstart-consumers
 x-mysql-profiles-dev: &mysql-profiles-dev
   - debug
+  - debug-min
   - debug-frontend
   - debug-backend
   - debug-consumers
@@ -26,6 +27,7 @@ x-mysql-profiles: &mysql-profiles
   - quickstart-storage
   - quickstart-consumers
   - debug
+  - debug-min
   - debug-datahub-actions
   - debug-frontend
   - debug-backend
@@ -65,6 +67,7 @@ x-opensearch-profiles-quickstart: &opensearch-profiles-quickstart
   - quickstart-consumers
 x-opensearch-profiles-dev: &opensearch-profiles-dev
   - debug
+  - debug-min
   - debug-datahub-actions
   - debug-frontend
   - debug-backend
@@ -82,6 +85,7 @@ x-opensearch-profiles: &opensearch-profiles
   - quickstart-postgres
   - quickstart-consumers
   - debug
+  - debug-min
   - debug-datahub-actions
   - debug-frontend
   - debug-backend
@@ -102,6 +106,7 @@ x-profiles-quickstart: &profiles-quickstart
   - quickstart-consumers
 x-profiles-dev: &profiles-dev
   - debug
+  - debug-min
   - debug-datahub-actions
   - debug-frontend
   - debug-backend

docs/developers.md

@@ -113,14 +113,43 @@ yarn install && yarn start
 
 The frontend will be available at `http://localhost:3000` and will automatically update as you make changes to the code.
 
-### Refreshing GMS
+### Refreshing components of quickStart
 
-To refresh the GMS (Generalized Metadata Service) with local changes:
+To refresh any of the running system stared by `./gradlew quickStartDebug`, run
 
 ```shell
-./gradlew :metadata-service:war:build -x test --parallel && docker restart datahub-datahub-gms-debug-1
+./gradlew debugReload
 ```
 
+This will build any changed components and restart those containers that had changes.
+There are a few other quickStart\* variants, like quickStartDebugMin, quickStartDebugConsumers
+
+For each of those variants, there is a corresponding reloadTask.
+
+For `./gradlew quickStartDebugConsumers`, the reload command is `./gradlew debugConsumersReload`
+For `./gradlew quickStartDebugMin`, the reload command is `./gradlew debugMinReload`
+
+A full restart using `./gradlew quickStartDebug` is recommended if there are significant changes and the setup/system update containers need to be run again.
+For incremental changes, the `debugReload*` variants can be used.
+
+### Using .env to configure settings of services started by quickstart
+
+To start datahub with a customized set of environment variables, .env files can be created in the docker/profiles folder.
+For example, an env file `my-settings.env` can be created in docker/profiles folder and loaded using
+
+```shell
+DATAHUB_LOCAL_COMMON_ENV=my-settings.env ./gradlew quickStartDebug
+```
+
+To refresh the containers due to code changes, `debugReload` task can be used.
+To change the env and reload containers, use the task `debugReloadEnv`
+
+```shell
+DATAHUB_LOCAL_COMMON_ENV=my-other-settings.env ./gradlew debugReloadEnv
+```
+
+This will build any container artifacts were changed and all reloadable containers are re-created to use the new env settings.
+
 ### Refreshing the CLI
 
 If you haven't set up the CLI for local development yet, run:
@@ -146,14 +175,6 @@ Expected Output:
 acryl-datahub, version unavailable (installed in develop mode)
 ```
 
-### Refreshing Other Components
-
-To refresh other components with local changes, just run:
-
-```commandline
-./gradlew quickstartDebug
-```
-
 ## IDE Support
 
 The recommended IDE for DataHub development is [IntelliJ IDEA](https://www.jetbrains.com/idea/).

docs/docker/development.md

@@ -40,31 +40,46 @@ acryldata/datahub-elasticsearch-setup            debug              4d935be7c62c
 
 At this point it is possible to view the DataHub UI at `http://localhost:9002` as you normally would with quickstart.
 
+Like `quickStartDebug`, there are a few other tasks that bring up a different set of containers, for example
+`quickStartDebugConsumers` will also bring up mce-consumer and mae-consumer.
+
 ## Reloading
 
-Next, perform the desired modifications and rebuild the frontend and/or GMS components.
+Next, perform the desired modifications
 
-**Builds GMS**
-
-```shell
-./gradlew :metadata-service:war:build
-```
-
-**Builds the frontend**
-
-Including javascript components.
-
-```shell
-./gradlew :datahub-frontend:build
-```
-
-After building the artifacts only a restart of the container(s) is required to run with the updated code.
-The restart can be performed using a docker UI, the docker cli, or the following gradle task.
+To see these changes in the deployment, a rebuilt of modified artifacts and a restart of the container(s) is required to run with the updated code.
+The restart can be performed using following gradle task.
 
 ```shell
 ./gradlew :docker:debugReload
 ```
 
+This single task will build the artifacts that were modified and restart only those containers that were affected by the rebuilt artifacts.
+
+For each of the `quickStartDebug` variants, there is a corresponding `debugReload` task.
+For `quickStartDebugConsumers`, the reload task is `debugConsumersReload`
+
+`debugReload` is generally much faster than re-running `quickStartDebug` and is recommended after an initial bringup of all services via `quickStartDebug` followed
+by loading the incremental changes using `debugReload`.
+
+If there are significant changes to the code, for example due to pulling the latest code, it is recommended to start with a `quickStartDebug` and then iterate using `debugReload`
+
+# Setting environment variables via env files
+
+You can define different sets of environment variables for all the containers in an env file. The env files must be located in the `docker/profiles` folder.
+To use the env file, run
+
+```shell
+DATAHUB_LOCAL_COMMON_ENV=my-settings.env ./gradlew quickStartDebug
+```
+
+The `debugReload` process continues to work, but the restarted containers will use the same settings that were present at the time of running `./gradlew quickStartDebug`.
+
+If you need to reload the containers with a different env file or changes made to the env file, a task `debugReloadEnv` builds the artifacts that have code changes
+and recreates all the containers that refer to these the env file via the DATAHUB_LOCAL_COMMON_ENV environment variable.
+
+`debugReloadEnv` also has variants for all the `quickStartDebug` variants. For example, `quickStartDebugConsumers` has `debugConsumersReloadEnv`
+
 ## Start/Stop
 
 The following commands can pause the debugging environment to release resources when not needed.