datahub/docker/build.gradle

plugins {
    id 'java' // required by versioning
    id 'docker-compose'
}
import com.avast.gradle.dockercompose.tasks.ComposeUp
import com.avast.gradle.dockercompose.tasks.ComposeDownForced

apply from: "../gradle/versioning/versioning.gradle"

ext {
    compose_base = "profiles/docker-compose.yml"
    project_name = "datahub"

    backend_profile_modules = [
            ':docker:elasticsearch-setup',
            ':docker:mysql-setup',
            ':docker:kafka-setup',
            ':datahub-upgrade',
            ':metadata-service:war',
    ]

    python_services_modules = [
    ]

    // Common configuration for all tasks
    common_config = [
            captureContainersOutput: true,
            captureContainersOutputToFiles: project.file('build/container-logs')
    ]

    // declarative task configuration
    quickstart_configs = [
            'quickstart': [
                    profile: 'quickstart-consumers',
                    modules: python_services_modules + backend_profile_modules + [
                            ':datahub-frontend',
                            ':metadata-jobs:mce-consumer-job',
                            ':metadata-jobs:mae-consumer-job',
                            ':datahub-actions',
                    ]
            ],
            'quickstartDebug': [
                    profile: 'debug',
                    modules: python_services_modules + backend_profile_modules + [':datahub-frontend', ':datahub-actions'],
                    isDebug: true
            ],
            'quickstartDebugConsumers': [
                    profile: 'debug-consumers',
                    modules: python_services_modules + backend_profile_modules + [':datahub-frontend',
                                                                          ':metadata-jobs:mce-consumer-job',
                                                                          ':metadata-jobs:mae-consumer-job',
                                                                          ':datahub-actions'],
                    isDebug: true
            ],
            'quickstartPg': [
                    profile: 'quickstart-postgres',
                    modules: (backend_profile_modules - [':docker:mysql-setup']) + [
                            ':docker:postgres-setup',
                            ':datahub-frontend',
                            ':datahub-actions',
                    ]
            ],
            'quickstartPgDebug': [
                    profile: 'debug-postgres',
                    modules: python_services_modules + (backend_profile_modules - [':docker:mysql-setup']) + [
                            ':docker:postgres-setup',
                            ':datahub-frontend'
                    ],
                    isDebug: true
            ],
            'quickstartSlim': [
                    profile: 'quickstart-backend',
                    modules: backend_profile_modules + [
                        ':docker:datahub-ingestion',
                    ],
                    additionalEnv: [
                            'DATAHUB_ACTIONS_IMAGE': 'acryldata/datahub-ingestion',
                            'ACTIONS_EXTRA_PACKAGES': 'acryl-datahub-actions[executor] acryl-datahub-actions',
                            'ACTIONS_CONFIG': 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml',
                            'DATAHUB_LOCAL_COMMON_ENV': "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env"
                    ]
            ],
            'quickstartStorage': [
                    profile: 'quickstart-storage',
                    preserveVolumes: true
            ],
            'allImages': [ //This is a special task just to include all images as dependencies - and is useful when CI needs to publish all images
                    profile: 'quickstart-consumers',
                    modules: python_services_modules + backend_profile_modules + [
                        ':datahub-frontend',
                        ':metadata-jobs:mce-consumer-job',
                        ':metadata-jobs:mae-consumer-job',
                        ':datahub-actions',
                        ':docker:datahub-ingestion'
                    ]
            ]
    ]

    //  only for debug variants of quickstart to enable <variant>Reload tasks. 
    // The actual service name needs the profile to be appended,  <container-name>-<profile>
    moduleToContainer = [
        ':metadata-service:war': 'datahub-gms',
        ':datahub-frontend': 'frontend',
        ':datahub-upgrade': 'system-update',
        ':metadata-jobs:mce-consumer-job': 'datahub-mce-consumer',
        ':metadata-jobs:mae-consumer-job': 'datahub-mae-consumer',
        ':datahub-actions': 'datahub-actions',
    ]
}

// Register all quickstart tasks
quickstart_configs.each { taskName, config ->
    tasks.register(taskName) {
        group = 'quickstart'
    }
    
}

// Dynamically create all quickstart tasks and configurations
dockerCompose {
    // Configure default settings that apply to all configurations
    useComposeFiles = [compose_base]
    projectName = project_name
    projectNamePrefix = ''
    buildBeforeUp = false
    buildBeforePull = false
    stopContainers = false
    removeVolumes = false

    quickstart_configs.each { taskName, config ->
        "${taskName}" {
            isRequiredBy(tasks.named(taskName))
            if (config.profile) {
                composeAdditionalArgs = ['--profile', config.profile]
            }

            // Common environment variables
            environment.put 'DATAHUB_VERSION', System.getenv("DATAHUB_VERSION") ?: "v${version}"
            environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false'
            environment.put "METADATA_TESTS_ENABLED", "true"
            environment.put "DATAHUB_REPO", "${docker_registry}"

            // Additional environment variables if specified
            if (config.additionalEnv) {
                config.additionalEnv.each { key, value ->
                    environment.put key, value
                }
            }

            useComposeFiles = [compose_base]
            projectName = project_name
            projectNamePrefix = ''
            buildBeforeUp = false
            buildBeforePull = false
            stopContainers = false
            removeVolumes = false

            // Apply common configuration
            common_config.each { key, value ->
                delegate."${key}" = value
            }

            // Apply additional task-specific configuration if specified
            if (config.additionalConfig) {
                config.additionalConfig.each { key, value ->
                    delegate."${key}" = value
                }
            }
        }
    }
}


// Register all quickstart tasks
quickstart_configs.each { taskName, config ->
    tasks.register("prepareAll${taskName}"){
        group = 'quickstart-ci'
    }
}

quickstart_configs.each { taskName, config ->
    if (config.modules) {
        tasks.getByName("prepareAll${taskName}").dependsOn(
            config.modules.collect { it + ':dockerPrepare' }
        )
    }
}


quickstart_configs.each { taskName, config ->
    tasks.register("buildImages${taskName}", Exec) {
        ext{
            bakeSpec = [:]
        }
        group = 'quickstart-ci'
        dependsOn(config.modules.collect { it + ':generateBakeSnippet' })
        dependsOn(tasks.getByName("prepareAll${taskName}"))

        def jsonFile = new File(rootProject.buildDir, "bake-spec-${taskName}.json")

        def bakeCmdArgs = ["bake", "-f",  "${jsonFile.absolutePath}"]
        def buildCmd = []
        if (System.getenv("DOCKER_CACHE") == "DEPOT") {
            buildCmd << "depot"
            buildCmd += bakeCmdArgs
            buildCmd += ['--save', '--metadata-file', "${rootProject.buildDir}/build-metadata.json"]
            if (project.properties.getOrDefault("dockerPush", false)){
                buildCmd << "--push"
            }
        } else {
            buildCmd += ["docker",  "buildx" ]
            buildCmd +=bakeCmdArgs
        }
        commandLine buildCmd
        workingDir rootProject.projectDir

        doFirst {
            def bakeSnippets = [:]
            def targets = []

            config.modules.each { module ->
                def moduleProject = project.project(module)
                def generateBakeSnippetsTask = moduleProject.tasks.getByName("generateBakeSnippet")
                bakeSnippets.putAll(generateBakeSnippetsTask.bakeSpec.target)
                targets.addAll(generateBakeSnippetsTask.bakeSpec.target.keySet())
            }

            ext.bakeSpec.group = [ "default": ["targets": targets] ]
            ext.bakeSpec.target = bakeSnippets

            jsonFile.parentFile.mkdirs()
            jsonFile.text = groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(ext.bakeSpec))
        }
    }
}

// Configure dependencies for ComposeUp tasks
quickstart_configs.each { taskName, config ->
    if (config.modules) {
        tasks.getByName("${taskName}ComposeUp").dependsOn(
                tasks.getByName("buildImages${taskName}")
        )
    }
}


tasks.register('minDockerCompose2.20', Exec) {
    executable 'bash'
    args '-c', 'echo -e "$(docker compose version --short)\n2.20"|sort --version-sort --check=quiet --reverse'
}

tasks.register('quickstartNuke') {
    group = 'quickstart'
    doFirst {
        quickstart_configs.each { taskName, config ->
            dockerCompose."${taskName}".removeVolumes = !config.preserveVolumes
        }
    }
    finalizedBy(tasks.withType(ComposeDownForced))
}

tasks.register('quickstartDown') {
    group = 'quickstart'
    finalizedBy(tasks.withType(ComposeDownForced))
}

tasks.withType(ComposeUp).configureEach {
    shouldRunAfter('quickstartNuke')
    dependsOn tasks.named("minDockerCompose2.20")
}

// Register all quickstart Reload tasks. For quickstartDebug, the reload task is DebugReload. (Taskname without quickstart prefix)
quickstart_configs.each { taskName, config ->
    if (config.isDebug) {
        def reloadTaskName = taskName.replaceFirst(/^quickstart/, "")
        tasks.register("${reloadTaskName}Reload", Exec) {
            dependsOn tasks.named("prepareAll${taskName}")
            group = 'quickstart'
            description = "Build and reload only changed containers for the ${taskName} task"
            doFirst {
                def executedTasks = project.gradle.taskGraph.allTasks.findAll { it.state.executed }
                def containersToRestart = []

                moduleToContainer.each { modulePath, containerName ->
                    def moduleProject = project.project(modulePath)
                    def dockerPrepareTask = moduleProject.tasks.findByName('dockerPrepare')
                    
                    if (dockerPrepareTask && executedTasks.contains(dockerPrepareTask) && !dockerPrepareTask.state.upToDate) {
                        containersToRestart << "${containerName}-${config.profile}"
                    }
                }
                
                // Only restart containers that had their modules rebuilt
                if (containersToRestart) {
                    def cmd = ["docker compose -p datahub --profile ${config.profile}"] + ['-f', compose_base] + ['restart'] + containersToRestart
                    println(cmd.join(" "))
                    commandLine 'bash', '-c', cmd.join(" ")
                } else {
                    // If no containers need restart, make this a no-op
                    commandLine 'bash', '-c', 'echo "No containers need restarting - all modules are up to date"'
                }    
            }
        }
    }
}