build: use versioning in gradle consistent with ci (#13259)

Enable use of gradle for all image builds for publishing, eliminating the per-image build action in docker-unified.yml that duplicated what was in gradle but used slightly different mechanisms to determine what is the tag. Enabled gradle build to consume tags provided by the workflow and produce tags same as earlier.

Use bake matrix builds to build slim/full versions of datahub-ingestion, datahub-actions.

Publish images and scan relies on gradle to get the list of images, via depot.

Image publish and scans run once a day on schedule or on manual triggers only.
Pending work: Separate the publish and scans into a separate workflow that runs on a schedule and could also run other tests.

build.gradle

@@ -65,7 +65,7 @@ buildscript {
   ext.logbackClassicJava8 = '1.2.12'
   ext.awsSdk2Version = '2.30.33'
 
-  ext.docker_registry = 'acryldata'
+  ext.docker_registry = project.getProperties().getOrDefault("dockerRegistry", 'acryldata')
 
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)

datahub-actions/build.gradle

@@ -28,7 +28,7 @@ ext {
   docker_registry = 'acryldata'
   docker_repo = 'datahub-actions'
   docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
-  docker_version = "v${version}${docker_target == 'slim' ? '-slim' : ''}"
+  
   python_docker_version = project.getProperties().getOrDefault("pythonDockerVersion", "1!0.0.0+docker.${version}")
 }
 
@@ -143,7 +143,7 @@ task cleanPythonCache(type: Exec) {
 
 docker {
   //dependsOn(build)
-  name "${docker_registry}/${docker_repo}:${versionTag}${docker_target == 'slim' ? '-slim' : ''}"
+  name "${docker_registry}/${docker_repo}:${versionTag}"
   dockerfile file("${rootProject.projectDir}/docker/datahub-actions/Dockerfile")
   files fileTree(rootProject.projectDir) {
     exclude "datahub-actions/scripts/**"
@@ -159,10 +159,15 @@ docker {
   }.exclude {
     i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
   }
-  buildArgs([APP_ENV: docker_target, RELEASE_VERSION: python_docker_version])
-
+  
   additionalTag("Debug", "${docker_registry}/${docker_repo}:debug")
 
+  defaultVariant = "slim"
+  variants = [
+    "slim": [suffix: "-slim", args: [APP_ENV: "slim", RELEASE_VERSION: python_docker_version]],
+    "full": [suffix: "", args: [APP_ENV: "full", RELEASE_VERSION: python_docker_version]]
+  ]
+
 }
 
 build.dependsOn install

docker/build.gradle

@@ -83,6 +83,16 @@ ext {
             'quickstartStorage': [
                     profile: 'quickstart-storage',
                     preserveVolumes: true
+            ],
+            'allImages': [ //This is a special task just to include all images as dependencies - and is useful when CI needs to publish all images
+                    profile: 'quickstart-consumers',
+                    modules: python_services_modules + backend_profile_modules + [
+                        ':datahub-frontend',
+                        ':metadata-jobs:mce-consumer-job',
+                        ':metadata-jobs:mae-consumer-job',
+                        ':datahub-actions',
+                        ':docker:datahub-ingestion'
+                    ]
             ]
     ]
 
@@ -194,6 +204,9 @@ quickstart_configs.each { taskName, config ->
             buildCmd << "depot"
             buildCmd += bakeCmdArgs
             buildCmd += ['--save', '--metadata-file', "${rootProject.buildDir}/build-metadata.json"]
+            if (project.properties.getOrDefault("dockerPush", false)){
+                buildCmd << "--push"
+            }
         } else {
             buildCmd += ["docker",  "buildx" ]
             buildCmd +=bakeCmdArgs
@@ -221,64 +234,6 @@ quickstart_configs.each { taskName, config ->
     }
 }
 
-//TODO This is a copy of buildImages* tasks but without the prepareAll dependency. Need to refactor to avoid code duplication.
-/*
-quickstart_configs.each { taskName, config ->
-    tasks.register("buildImagesFromCache${taskName}", Exec) {
-        ext{
-            bakeSpec = [:]
-        }
-        group = 'quickstart-ci'
-        dependsOn(config.modules.collect { it + ':generateBakeSnippet' })
-
-        def jsonFile = new File(rootProject.buildDir, "bake-spec-${taskName}.json")
-
-        bakeCmdArgs = ["bake", "-f",  "${jsonFile.absolutePath}"]
-        def buildCmd = []
-        if (System.getenv("DOCKER_CACHE") == "GITHUB") {
-            buildCmd += ["docker",  "buildx" ]
-            buildCmd += bakeCmdArgs
-            def githubToken = System.getenv("GITHUB_TOKEN")
-            if (githubToken) {
-                dockerCmd += ["--cache-from", "type=gha,token=${githubToken}"]
-                dockerCmd += ["--cache-to", "type=gha,mode=max,token=${githubToken}"]
-            } else {
-                dockerCmd += ["--cache-from", "type=gha"]
-                dockerCmd += ["--cache-to", "type=gha,mode=max"]
-            }
-        } else if (System.getenv("DOCKER_CACHE") == "DEPOT") {
-            buildCmd << "depot"
-            buildCmd += bakeCmdArgs
-            buildCmd += ['--save', '--metadata-file', "${rootProject.buildDir}/build-metadata.json"]
-        } else {
-            buildCmd += ["docker",  "buildx" ]
-            buildCmd +=bakeCmdArgs
-        }
-        println(buildCmd.join(" "))
-        commandLine buildCmd
-        workingDir rootProject.projectDir
-
-        doFirst {
-            def bakeSnippets = [:]
-            def targets = []
-
-            config.modules.each { module ->
-                def moduleProject = project.project(module)
-                def generateBakeSnippetsTask = moduleProject.tasks.getByName("generateBakeSnippet")
-                bakeSnippets.putAll(generateBakeSnippetsTask.bakeSpec.target)
-                targets.addAll(generateBakeSnippetsTask.bakeSpec.target.keySet())
-            }
-
-            ext.bakeSpec.group = [ "default": ["targets": targets] ]
-            ext.bakeSpec.target = bakeSnippets
-
-            jsonFile.parentFile.mkdirs()
-            jsonFile.text = groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(ext.bakeSpec))
-        }
-    }
-}
-*/
-
 // Configure dependencies for ComposeUp tasks
 quickstart_configs.each { taskName, config ->
     if (config.modules) {

docker/datahub-gms/Dockerfile

@@ -63,7 +63,7 @@ FROM base AS dev-install
 
 FROM ${APP_ENV}-install AS final
 
-RUN mkdir -p /etc/datahub/plugins/auth/resources
+RUN mkdir -p /etc/datahub/plugins/auth/resources /datahub
 
 RUN addgroup -S datahub && adduser -S datahub -G datahub && chmod g-s /home/datahub
 RUN chown -R datahub:datahub /etc/datahub /datahub

docker/datahub-ingestion/build.gradle

@@ -10,7 +10,7 @@ ext {
     docker_repo = 'datahub-ingestion'
     docker_dir = 'datahub-ingestion'
     docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
-    docker_version = "v${version}${docker_target == 'slim' ? '-slim' : ''}"
+    docker_version = "${versionTag}"  // Variants will add suffix to this tag
     python_docker_version = project.getProperties().getOrDefault("pythonDockerVersion", "1!0.0.0+docker.${version}")
 
     revision = 12 // increment to trigger rebuild
@@ -36,9 +36,16 @@ docker {
     }
 
     version "${docker_version}"
+
+    defaultVariant = "slim"
+
+    variants = [
+        "slim": [suffix: "", args: [RELEASE_VERSION: python_docker_version, APP_ENV: "slim"]],
+        "full": [suffix: "full", args: [RELEASE_VERSION: python_docker_version, APP_ENV: "full"]]
+        ]
     // This task is intended to build the slim image
-    target 'ingestion-base-slim'
-    def dockerBuildArgs = [RELEASE_VERSION: python_docker_version, APP_ENV: docker_target]
+    //target 'ingestion-base-slim' //Review if this needs to be handled by bake
+    def dockerBuildArgs = [RELEASE_VERSION: python_docker_version]
 
     // Add build args if they are defined (needed for some CI or enterprise environments)
     if (project.hasProperty('pipMirrorUrl')) {

gradle/docker/docker.gradle

@@ -53,6 +53,15 @@ class DockerPluginExtension {
   ListProperty<String> platforms
   ListProperty<Object> dependencies // May contain tasks or task  names
   Property<String> target
+  MapProperty<String, Map<String, String>> variants
+  
+  Property<String> defaultVariant
+  List<String> defaultPlatforms = ["linux/amd64", "linux/arm64/v8"]
+
+  // For quickStart debug builds that use APP_ENV=dev like pattern. Not used in matrix builds.
+  MapProperty<String, String> debugBuildArgs
+
+  
 
   DockerPluginExtension(Project project) {
     this.project = project
@@ -63,6 +72,13 @@ class DockerPluginExtension {
     platforms = project.objects.listProperty(String)
     dependencies = project.objects.listProperty(Object)
     target = project.objects.property(String)
+    variants = project.objects.mapProperty(String, Map)
+    // quickstart builds does not require all variants to be built. So, if default variant is specified, and 
+    // projectProperty matrixBuild is not set to true, then only default variant is built to save time. 
+    // The defaultVariant does not use a suffix in the tag.
+    defaultVariant = project.objects.property(String)
+    
+    debugBuildArgs = project.objects.mapProperty(String, String)
   }
 
   def files(Object... files) {
@@ -81,6 +97,10 @@ class DockerPluginExtension {
     buildArgs.putAll(values)
   }
 
+  def debugBuildArgs(Map<String, String> values) {
+    debugBuildArgs.putAll(values)
+  }
+
   def platform(String... platforms) {
     this.platforms.addAll(platforms)
   }
@@ -229,21 +249,73 @@ project.afterEvaluate {
     description "Generates bake snippets for the project"
 
     doLast {
+      // if matrixBuild is true, this is to publish images, so all variants must be built.
+      def matrixBuild = project.getProperties().getOrDefault("matrixBuild", false)
+
       def bake_spec_target = [
         context: "${buildContext}",
         dockerfile: "${extension.dockerfile.get().toPath()}",
-        tags: extension.tags.get().values()
       ]
+
+      if (project.hasProperty("tag")) {
+        def dockerTag = project.property("tag")
+        bake_spec_target.tags = extension.tags.get().values().findAll({ tag -> tag.contains(dockerTag) }).toList()
+      } else {
+        bake_spec_target.tags = extension.tags.get().values().toList()
+      }
       if (extension.buildArgs.get()) {
         bake_spec_target.args = extension.buildArgs.get()
-      }
-      if (extension.platforms.get()) {
-        bake_spec_target.platforms = extension.platforms.get()
+      } else {
+        bake_spec_target.args = [:]
       }
 
+      if (extension.debugBuildArgs.get() && bake_spec_target.tags.findAll { tag -> tag.contains("debug") }.size() > 0) {
+        bake_spec_target.args.putAll(extension.debugBuildArgs.get())
+      }
+
+      if (matrixBuild) {
+        if (extension.platforms.get()) {
+          bake_spec_target.platforms = extension.platforms.get()
+        } else {
+          bake_spec_target.platforms = extension.defaultPlatforms
+        }
+      }
+
+      if (extension.variants.get()) {
+        bake_spec_target.name = "${project.name}-\${variants.tagSuffix}"
+        bake_spec_target.matrix = [variants: []]
+
+        extension.variants.get().each { variant, variantSpec ->
+          if (matrixBuild || variant == extension.defaultVariant.get()) {
+              // Its not easy to merge common buildArgs and variant buildArgs with json format. 
+              // So, we just add common buildArgs to matrix.
+              bake_spec_target.matrix.variants.add([ 'tagSuffix': "${variantSpec.suffix}", 'args': variantSpec.args + bake_spec_target.args])
+            }
+        }
+        // When variants are present, fix up tags to include the variant suffix
+        // A bit of a hack that for quickstart builds, we need a tag named debug that contains the default variant without suffixes
+        bake_spec_target.tags = bake_spec_target.tags.collect(
+            { tag -> tag + (tag.contains("debug") ? "" : "\${variants.tagSuffix}") }
+          ).toList()
+        if (project.hasProperty("shaTag")) {
+          def shaTag = project.property("shaTag")
+          def dockerRepo = extension.tags.get().get("").split(":")[0] //Extract the repo name from the default tag
+          bake_spec_target.tags.add("${dockerRepo}:${shaTag}\${variants.tagSuffix}")
+        }
+
+        bake_spec_target.args = "\${variants.args}"
+
+      } else {
+        if (project.hasProperty("shaTag")) {
+          def shaTag = project.property("shaTag")
+          def dockerRepo = extension.tags.get().get("").split(":")[0]
+          bake_spec_target.tags.add("${dockerRepo}:${shaTag}")
+        }
+      }
       ext.bakeSpec = [
         target: [ "${project.name}": bake_spec_target]
       ]
+
     }
   }
 

gradle/versioning/versioning-global.gradle

@@ -23,14 +23,15 @@ def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
 def cliMajorVersion = "0.15.0" // base default cli major version
 def snapshotVersion = false
 def javaVersion = ""
-// tag for docker images. the prefix v is used in tag only if the tag is computed from a version. If a releaseVersion is supplied, use it as is.
-// This enables pr tags to be used without the v prefix. This variance was previouslyhandled in the CI steps build images without using gradle.
+
+// Used to tag docker images. If the project property tag is set, use it as is (posssibly add suffixes like slim, pythonVersion if applicable). 
+// Otherwise, compute the tag from the version.
 def versionTag = "" 
 
+
 if (project.hasProperty("releaseVersion")) {
   version = releaseVersion
   detailedVersionString = releaseVersion
-  versionTag = releaseVersion
 } else {
   try {
       // apply this plugin in a try-catch block so that we can handle cases without .git directory
@@ -39,7 +40,6 @@ if (project.hasProperty("releaseVersion")) {
       detailedVersionString = gitVersion()
       version = details.lastTag
       version = version.startsWith("v")? version.substring(1): version
-      versionTag = "v" + version
       def suffix = details.isCleanTag? "": "-SNAPSHOT"
       snapshotVersion = ! details.isCleanTag
     }
@@ -82,10 +82,15 @@ if (project.hasProperty("releaseVersion")) {
         // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
         javaVersion = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
       }
-      versionTag = "v" + version
   }
 }
 
+if (project.hasProperty("tag")) {
+  versionTag = tag
+} else {
+  versionTag = "v" + version
+}
+
 // Note: No task, we want this executed during config phase, once for rootProject.
 def data = [
     fullVersion: detailedVersionString,
@@ -105,4 +110,9 @@ rootProject.buildDir.mkdirs()
 // Write to file
 outputFile.text = jsonBuilder.toPrettyString()
 
-println "git.properties JSON data written to ${outputFile}"
+task printVersionDetails() {
+  println("fullVersion=" + detailedVersionString)
+  println("cliMajorVersion=" + cliMajorVersion)
+  println("version=" +  version)
+  println("versionTag=" + versionTag)
+}

metadata-service/war/build.gradle

@@ -120,6 +120,8 @@ docker {
   if (dockerBuildArgs.size() > 0) {
     buildArgs(dockerBuildArgs)
   }
+
+  debugBuildArgs = [APP_ENV:  'dev']
 }
 
 test {