mirror of
https://github.com/datahub-project/datahub.git
synced 2025-06-27 05:03:31 +00:00
347 lines
14 KiB
Groovy
347 lines
14 KiB
Groovy
plugins {
|
|
id 'java' // required by versioning
|
|
id 'docker-compose'
|
|
}
|
|
import com.avast.gradle.dockercompose.tasks.ComposeUp
|
|
import com.avast.gradle.dockercompose.tasks.ComposeDownForced
|
|
|
|
apply from: "../gradle/versioning/versioning.gradle"
|
|
|
|
ext {
|
|
compose_base = "profiles/docker-compose.yml"
|
|
project_name = "datahub"
|
|
|
|
backend_profile_modules = [
|
|
':docker:elasticsearch-setup',
|
|
':docker:mysql-setup',
|
|
':docker:kafka-setup',
|
|
':datahub-upgrade',
|
|
':metadata-service:war',
|
|
]
|
|
|
|
python_services_modules = [
|
|
]
|
|
|
|
// Common configuration for all tasks
|
|
common_config = [
|
|
captureContainersOutput: true,
|
|
captureContainersOutputToFiles: project.file('build/container-logs')
|
|
]
|
|
|
|
// declarative task configuration
|
|
quickstart_configs = [
|
|
'quickstart': [
|
|
profile: 'quickstart-consumers',
|
|
modules: python_services_modules + backend_profile_modules + [
|
|
':datahub-frontend',
|
|
':metadata-jobs:mce-consumer-job',
|
|
':metadata-jobs:mae-consumer-job',
|
|
':datahub-actions',
|
|
]
|
|
],
|
|
'quickstartDebug': [
|
|
profile: 'debug',
|
|
modules: python_services_modules + backend_profile_modules + [':datahub-frontend', ':datahub-actions'],
|
|
isDebug: true,
|
|
additionalEnv: [
|
|
DATAHUB_LOCAL_ACTIONS_ENV: "${rootProject.project(':smoke-test').projectDir}/test_resources/actions/actions.env"
|
|
]
|
|
],
|
|
'quickstartDebugMin': [
|
|
profile: 'debug-min',
|
|
modules: backend_profile_modules + [':datahub-frontend'],
|
|
isDebug: true
|
|
],
|
|
'quickstartDebugConsumers': [
|
|
profile: 'debug-consumers',
|
|
modules: python_services_modules + backend_profile_modules + [':datahub-frontend',
|
|
':metadata-jobs:mce-consumer-job',
|
|
':metadata-jobs:mae-consumer-job',
|
|
':datahub-actions'],
|
|
isDebug: true,
|
|
additionalEnv: [
|
|
DATAHUB_LOCAL_ACTIONS_ENV: "${rootProject.project(':smoke-test').projectDir}/test_resources/actions/actions.env"
|
|
]
|
|
],
|
|
'quickstartPg': [
|
|
profile: 'quickstart-postgres',
|
|
modules: (backend_profile_modules - [':docker:mysql-setup']) + [
|
|
':docker:postgres-setup',
|
|
':datahub-frontend',
|
|
':datahub-actions',
|
|
]
|
|
],
|
|
'quickstartPgDebug': [
|
|
profile: 'debug-postgres',
|
|
modules: python_services_modules + (backend_profile_modules - [':docker:mysql-setup']) + [
|
|
':docker:postgres-setup',
|
|
':datahub-frontend'
|
|
],
|
|
isDebug: true
|
|
],
|
|
'quickstartSlim': [
|
|
profile: 'quickstart-backend',
|
|
modules: backend_profile_modules + [
|
|
':datahub-actions',
|
|
]
|
|
],
|
|
'quickstartSpark': [
|
|
profile: 'quickstart-backend',
|
|
modules: backend_profile_modules + [
|
|
':datahub-actions',
|
|
],
|
|
additionalEnv: [
|
|
'DATAHUB_LOCAL_COMMON_ENV': "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env"
|
|
]
|
|
],
|
|
'quickstartStorage': [
|
|
profile: 'quickstart-storage',
|
|
preserveVolumes: true
|
|
],
|
|
'quickstartBackendDebug': [
|
|
profile: 'debug-backend-aws',
|
|
modules: python_services_modules + backend_profile_modules + [':datahub-frontend', ':datahub-actions'],
|
|
isDebug: true,
|
|
additionalEnv: [
|
|
DATAHUB_LOCAL_ACTIONS_ENV: "${rootProject.project(':smoke-test').projectDir}/test_resources/actions/actions.env"
|
|
]
|
|
],
|
|
'allImages': [ //This is a special task just to include all images as dependencies - and is useful when CI needs to publish all images
|
|
profile: 'quickstart-consumers',
|
|
modules: python_services_modules + backend_profile_modules + [
|
|
':datahub-frontend',
|
|
':metadata-jobs:mce-consumer-job',
|
|
':metadata-jobs:mae-consumer-job',
|
|
':datahub-actions',
|
|
':docker:datahub-ingestion',
|
|
':docker:postgres-setup'
|
|
]
|
|
]
|
|
]
|
|
|
|
// only for debug variants of quickstart to enable <variant>Reload tasks.
|
|
// The actual service name needs the profile to be appended, <container-name>-<profile>
|
|
// This list only contains modules that can be reloaded via the reloadTask. If other modules need to be reloaded, quickstart* needs to be used.
|
|
moduleToContainer = [
|
|
':metadata-service:war': 'datahub-gms',
|
|
':datahub-frontend': 'frontend',
|
|
':metadata-jobs:mce-consumer-job': 'datahub-mce-consumer',
|
|
':metadata-jobs:mae-consumer-job': 'datahub-mae-consumer',
|
|
]
|
|
}
|
|
|
|
// Register all quickstart tasks
|
|
quickstart_configs.each { taskName, config ->
|
|
tasks.register(taskName) {
|
|
group = 'quickstart'
|
|
}
|
|
|
|
}
|
|
|
|
// Dynamically create all quickstart tasks and configurations
|
|
dockerCompose {
|
|
// Configure default settings that apply to all configurations
|
|
useComposeFiles = [compose_base]
|
|
projectName = project_name
|
|
projectNamePrefix = ''
|
|
buildBeforeUp = false
|
|
buildBeforePull = false
|
|
stopContainers = false
|
|
removeVolumes = false
|
|
|
|
quickstart_configs.each { taskName, config ->
|
|
"${taskName}" {
|
|
isRequiredBy(tasks.named(taskName))
|
|
if (config.profile) {
|
|
composeAdditionalArgs = ['--profile', config.profile]
|
|
}
|
|
|
|
// Common environment variables
|
|
environment.put 'DATAHUB_VERSION', System.getenv("DATAHUB_VERSION") ?: "v${version}"
|
|
environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false'
|
|
environment.put "METADATA_TESTS_ENABLED", "true"
|
|
environment.put "DATAHUB_REPO", "${docker_registry}"
|
|
|
|
// Additional environment variables if specified
|
|
if (config.additionalEnv) {
|
|
config.additionalEnv.each { key, value ->
|
|
environment.put key, value
|
|
}
|
|
}
|
|
|
|
useComposeFiles = [compose_base]
|
|
projectName = project_name
|
|
projectNamePrefix = ''
|
|
buildBeforeUp = false
|
|
buildBeforePull = false
|
|
stopContainers = false
|
|
removeVolumes = false
|
|
retainContainersOnStartupFailure = config.isDebug ? true: false //forcing nulls to bool
|
|
|
|
// Apply common configuration
|
|
common_config.each { key, value ->
|
|
delegate."${key}" = value
|
|
}
|
|
|
|
// Apply additional task-specific configuration if specified
|
|
if (config.additionalConfig) {
|
|
config.additionalConfig.each { key, value ->
|
|
delegate."${key}" = value
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
// Register all quickstart tasks
|
|
quickstart_configs.each { taskName, config ->
|
|
tasks.register("prepareAll${taskName}"){
|
|
group = 'quickstart-ci'
|
|
}
|
|
}
|
|
|
|
quickstart_configs.each { taskName, config ->
|
|
if (config.modules) {
|
|
tasks.getByName("prepareAll${taskName}").dependsOn(
|
|
config.modules.collect { it + ':dockerPrepare' }
|
|
)
|
|
}
|
|
}
|
|
|
|
|
|
quickstart_configs.each { taskName, config ->
|
|
tasks.register("buildImages${taskName}", Exec) {
|
|
ext{
|
|
bakeSpec = [:]
|
|
}
|
|
group = 'quickstart-ci'
|
|
def taskSuffix = config.isDebug? 'debug' : ''
|
|
dependsOn(config.modules.collect { it + ":generateBakeSnippet${taskSuffix}" })
|
|
dependsOn(tasks.getByName("prepareAll${taskName}"))
|
|
|
|
def jsonFile = new File(rootProject.buildDir, "bake-spec-${taskName}.json")
|
|
|
|
def bakeCmdArgs = ["bake", "-f", "${jsonFile.absolutePath}"]
|
|
def buildCmd = []
|
|
if (System.getenv("DOCKER_CACHE") == "DEPOT") {
|
|
buildCmd << "depot"
|
|
buildCmd += bakeCmdArgs
|
|
buildCmd += ['--save', '--metadata-file', "${rootProject.buildDir}/build-metadata.json"]
|
|
if (project.properties.getOrDefault("dockerPush", false)){
|
|
buildCmd << "--push"
|
|
}
|
|
} else {
|
|
buildCmd += ["docker", "buildx" ]
|
|
buildCmd +=bakeCmdArgs
|
|
}
|
|
commandLine buildCmd
|
|
workingDir rootProject.projectDir
|
|
|
|
doFirst {
|
|
def bakeSnippets = [:]
|
|
def targets = []
|
|
|
|
config.modules.each { module ->
|
|
def moduleProject = project.project(module)
|
|
def generateBakeSnippetsTask = moduleProject.tasks.getByName("generateBakeSnippet${taskSuffix}")
|
|
bakeSnippets.putAll(generateBakeSnippetsTask.bakeSpec.target)
|
|
targets.addAll(generateBakeSnippetsTask.bakeSpec.target.keySet())
|
|
}
|
|
|
|
ext.bakeSpec.group = [ "default": ["targets": targets] ]
|
|
ext.bakeSpec.target = bakeSnippets
|
|
|
|
jsonFile.parentFile.mkdirs()
|
|
jsonFile.text = groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(ext.bakeSpec))
|
|
}
|
|
}
|
|
}
|
|
|
|
// Configure dependencies for ComposeUp tasks
|
|
quickstart_configs.each { taskName, config ->
|
|
if (config.modules) {
|
|
tasks.getByName("${taskName}ComposeUp").dependsOn(
|
|
tasks.getByName("buildImages${taskName}")
|
|
)
|
|
}
|
|
}
|
|
|
|
|
|
tasks.register('minDockerCompose2.20', Exec) {
|
|
executable 'bash'
|
|
args '-c', 'echo -e "$(docker compose version --short)\n2.20"|sort --version-sort --check=quiet --reverse'
|
|
}
|
|
|
|
tasks.register('quickstartNuke') {
|
|
group = 'quickstart'
|
|
doFirst {
|
|
quickstart_configs.each { taskName, config ->
|
|
dockerCompose."${taskName}".removeVolumes = !config.preserveVolumes
|
|
}
|
|
}
|
|
finalizedBy(tasks.withType(ComposeDownForced))
|
|
}
|
|
|
|
tasks.register('quickstartDown') {
|
|
group = 'quickstart'
|
|
finalizedBy(tasks.withType(ComposeDownForced))
|
|
}
|
|
|
|
tasks.withType(ComposeUp).configureEach {
|
|
shouldRunAfter('quickstartNuke')
|
|
dependsOn tasks.named("minDockerCompose2.20")
|
|
}
|
|
|
|
// Register all quickstart Reload tasks. For quickstartDebug, the reload task is DebugReload. (Taskname without quickstart prefix)
|
|
quickstart_configs.each { taskName, config ->
|
|
if (config.isDebug) {
|
|
def reloadTaskName = taskName.replaceFirst(/^quickstart/, "")
|
|
tasks.register("${reloadTaskName}Reload", Exec) {
|
|
dependsOn tasks.named("prepareAll${taskName}")
|
|
group = 'quickstart'
|
|
description = "Build and reload only changed containers for the ${taskName} task"
|
|
doFirst {
|
|
def executedTasks = project.gradle.taskGraph.allTasks.findAll { it.state.executed }
|
|
def containersToRestart = []
|
|
|
|
moduleToContainer.each { modulePath, containerName ->
|
|
def moduleProject = project.project(modulePath)
|
|
def dockerPrepareTask = moduleProject.tasks.findByName('dockerPrepare')
|
|
|
|
if (dockerPrepareTask && executedTasks.contains(dockerPrepareTask) && !dockerPrepareTask.state.upToDate) {
|
|
containersToRestart << "${containerName}-${config.profile}"
|
|
}
|
|
}
|
|
|
|
// Only restart containers that had their modules rebuilt
|
|
if (containersToRestart) {
|
|
def cmd = ["docker compose -p datahub --profile ${config.profile}"] + ['-f', compose_base] + ['restart'] + containersToRestart
|
|
println(cmd.join(" "))
|
|
commandLine 'bash', '-c', cmd.join(" ")
|
|
} else {
|
|
// If no containers need restart, make this a no-op
|
|
commandLine 'bash', '-c', 'echo "No containers need restarting - all modules are up to date"'
|
|
}
|
|
}
|
|
}
|
|
tasks.register("${reloadTaskName}ReloadEnv", Exec) {
|
|
dependsOn tasks.named("prepareAll${taskName}")
|
|
group = 'quickstart'
|
|
description = "Build changed containers but recreate all services for the ${taskName} task"
|
|
doFirst {
|
|
def containersToRestart = []
|
|
moduleToContainer.each { modulePath, containerName ->
|
|
// Find which of of the reloadable modules are in used in this task
|
|
if (config.modules.contains(modulePath)) {
|
|
containersToRestart << "${containerName}-${config.profile}"
|
|
}
|
|
}
|
|
def cmd = ["docker compose -p datahub --profile ${config.profile}"] + ['-f', compose_base] + ['up', '-d', '--no-deps'] + containersToRestart
|
|
println(cmd.join(" "))
|
|
commandLine 'bash', '-c', cmd.join(" ")
|
|
}
|
|
}
|
|
}
|
|
}
|