fix(build): build improvements to help with incremental builds (#12823)

build.gradle

@@ -1,3 +1,6 @@
+import org.apache.tools.ant.filters.ReplaceTokens
+
+
 buildscript {
   ext.jdkVersionDefault = 17
   ext.javaClassVersionDefault = 11
@@ -396,25 +399,56 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
   }
 }
 
-subprojects {
-
-  apply plugin: 'maven-publish'
-  apply plugin: 'com.gorylenko.gradle-git-properties'
-  apply plugin: 'com.diffplug.spotless'
-
-  gitProperties {
+apply plugin: 'com.gorylenko.gradle-git-properties'
+gitProperties {
   keys = ['git.commit.id','git.commit.id.describe','git.commit.time']
   // using any tags (not limited to annotated tags) for "git.commit.id.describe" property
   // see http://ajoberstar.org/grgit/grgit-describe.html for more info about the describe method and available parameters
   // 'it' is an instance of org.ajoberstar.grgit.Grgit
   customProperty 'git.commit.id.describe', { it.describe(tags: true) }
+  gitPropertiesResourceDir = rootProject.buildDir
   failOnNoGitDirectory = false
+}
+
+def gitPropertiesGenerated = false
+
+apply from: 'gradle/versioning/versioning-global.gradle'
+
+tasks.register("generateGitPropertiesGlobal", com.gorylenko.GenerateGitPropertiesTask) {
+  doFirst {
+    if (!gitPropertiesGenerated) {
+      println "Generating git.properties"
+      gitPropertiesGenerated = true
+    } else {
+      // Skip actual execution if already run
+      onlyIf { false }
+    }
+  }
+}
+
+subprojects {
+
+  apply plugin: 'maven-publish'
+  apply plugin: 'com.diffplug.spotless'
+
+  def gitPropertiesTask = tasks.register("copyGitProperties", Copy) {
+    dependsOn rootProject.tasks.named("generateGitPropertiesGlobal")
+    def sourceFile = file("${rootProject.buildDir}/git.properties")
+    from sourceFile
+    into "$project.buildDir/resources/main"
   }
 
   plugins.withType(JavaPlugin).configureEach {
+    project.tasks.named(JavaPlugin.CLASSES_TASK_NAME).configure{
+      dependsOn gitPropertiesTask
+    }
     if (project.name == 'datahub-web-react') {
       return
     }
+    /* TODO: evaluate ignoring jar timestamps for increased caching (compares checksum instead)
+    jar {
+      preserveFileTimestamps = false
+    }*/
 
     dependencies {
       implementation externalDependency.annotationApi

datahub-web-react/build.gradle

@@ -111,14 +111,13 @@ task yarnBuild(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
   outputs.dir('dist')
 }
 
-task cleanExtraDirs {
+clean {
   delete 'node_modules/.yarn-integrity'
   delete 'dist'
   delete 'tmp'
   delete 'just'
   delete fileTree(dir: 'src', include: '*.generated.ts')
 }
-clean.finalizedBy(cleanExtraDirs)
 
 configurations {
   assets

gradle/versioning/versioning-global.gradle

@@ -0,0 +1,101 @@
+/**
+ Applies a consistent versioning scheme to all projects using this script
+
+Uses git tags to mint versions by default. 
+git tags can be of a few forms:
+- short sha (typical for a PR or a commit) (e.g. 38960ae)
+- versioned tags (typical for a release) (e.g. v0.8.45, v0.8.45.1, v0.8.45rc1, v0.8.45.1rc4)
+
+Produces the following variables and supports token replacement
+- version: server version amenable for creating jars
+- fullVersion: full version string 
+- cliMajorVersion: cli version amenable for binding to server as a default
+  0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories)
+
+ All inference can be overridden by passing in the releaseVersion property
+ e.g. -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4
+
+ **/
+
+import groovy.json.JsonBuilder
+
+def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
+def cliMajorVersion = "0.15.0" // base default cli major version
+def snapshotVersion = false
+def javaVersion = ""
+
+if (project.hasProperty("releaseVersion")) {
+  version = releaseVersion
+  detailedVersionString = releaseVersion
+} else {
+  try {
+      // apply this plugin in a try-catch block so that we can handle cases without .git directory
+      apply plugin: "com.palantir.git-version"
+      def details = versionDetails()
+      detailedVersionString = gitVersion()
+      version = details.lastTag
+      version = version.startsWith("v")? version.substring(1): version
+      def suffix = details.isCleanTag? "": "-SNAPSHOT"
+      snapshotVersion = ! details.isCleanTag
+    }
+    catch (Exception e) {
+          e.printStackTrace()
+          // last fall back
+          version = detailedVersionString
+    }
+}
+
+// trim version if it is of size 4 to size 3
+def versionParts = version.tokenize(".")
+cliMajorVersion = version
+if (versionParts.size() > 3) {
+  // at-least 4 part version
+  // we check if the 4th part is a .0 in which case we want to create a release
+  if ((versionParts.size() == 4) && (versionParts[3] == '0')) {
+    versionParts = versionParts[0..2]
+  }
+  version = versionParts[0..2].join('.')
+  if (versionParts.size() > 3) {
+      version = version + "-" + versionParts[3..versionParts.size()-1].join('-')
+  }
+}
+
+if (snapshotVersion) {
+    if (versionParts[versionParts.size()-1].isInteger()) {
+      def base_version = versionParts[0..versionParts.size()-2].join('.')
+      version = base_version + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
+      cliMajorVersion = base_version + "." + versionParts[versionParts.size()-1]
+    } else {
+      // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
+      version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
+      cliMajorVersion = versionParts[0..versionParts.size()-1].join('.')
+    }
+
+    // differences from metadata-integration/java/versioning.gradle
+    if (versionParts[versionParts.size()-1].isInteger()) {
+      javaVersion = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
+    } else {
+      // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
+      javaVersion = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
+    }
+}
+
+// Note: No task, we want this executed during config phase, once for rootProject.
+def data = [
+    fullVersion: detailedVersionString,
+    cliMajorVersion: cliMajorVersion,
+    version: version,
+    javaVersion: javaVersion
+]
+
+// Convert to JSON
+def jsonBuilder = new JsonBuilder(data)
+def outputFile = file("${rootProject.buildDir}/version.json")
+
+// Ensure buildDir exists
+rootProject.buildDir.mkdirs()
+
+// Write to file
+outputFile.text = jsonBuilder.toPrettyString()
+
+println "git.properties JSON data written to ${outputFile}"

gradle/versioning/versioning.gradle

@@ -1,83 +1,33 @@
-/**
- Applies a consistent versioning scheme to all projects using this script
-
-Uses git tags to mint versions by default. 
-git tags can be of a few forms:
-- short sha (typical for a PR or a commit) (e.g. 38960ae)
-- versioned tags (typical for a release) (e.g. v0.8.45, v0.8.45.1, v0.8.45rc1, v0.8.45.1rc4)
-
-Produces the following variables and supports token replacement
-- version: server version amenable for creating jars
-- fullVersion: full version string 
-- cliMajorVersion: cli version amenable for binding to server as a default
-  0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories)
-
- All inference can be overridden by passing in the releaseVersion property
- e.g. -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4
-
- **/
-
-
+import groovy.json.JsonSlurper
 import org.apache.tools.ant.filters.ReplaceTokens
 
+
 def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
 def cliMajorVersion = "0.15.0" // base default cli major version
-def snapshotVersion = false
-if (project.hasProperty("releaseVersion")) {
-  version = releaseVersion
-  detailedVersionString = releaseVersion
-} else {
-  try {
-      // apply this plugin in a try-catch block so that we can handle cases without .git directory
-      apply plugin: "com.palantir.git-version"
-      def details = versionDetails()
-      detailedVersionString = gitVersion()
-      version = details.lastTag
-      version = version.startsWith("v")? version.substring(1): version
-      def suffix = details.isCleanTag? "": "-SNAPSHOT"
-      snapshotVersion = ! details.isCleanTag
-    }
-    catch (Exception e) {
-          e.printStackTrace()
-          // last fall back
-          version = detailedVersionString
-    }
-}
 
-// trim version if it is of size 4 to size 3
-def versionParts = version.tokenize(".")
-cliMajorVersion = version
-if (versionParts.size() > 3) {
-  // at-least 4 part version
-  // we check if the 4th part is a .0 in which case we want to create a release
-  if ((versionParts.size() == 4) && (versionParts[3] == '0')) {
-    versionParts = versionParts[0..2]
-  }
-  version = versionParts[0..2].join('.')
-  if (versionParts.size() > 3) {
-      version = version + "-" + versionParts[3..versionParts.size()-1].join('-')
-  }
-}
+def inputFile = file("${rootProject.buildDir}/version.json")
 
-if (snapshotVersion) {
-    if (versionParts[versionParts.size()-1].isInteger()) {
-      def base_version = versionParts[0..versionParts.size()-2].join('.')
-      version = base_version + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
-      cliMajorVersion = base_version + "." + versionParts[versionParts.size()-1]
+task readJsonData {
+  if (inputFile.exists()) {
+    def jsonSlurper = new JsonSlurper()
+    def data = jsonSlurper.parse(inputFile)
+
+    detailedVersionString = data.fullVersion
+    cliMajorVersion = data.cliMajorVersion
+    version = data.version
   } else {
-      // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
-      version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
-      cliMajorVersion = versionParts[0..versionParts.size()-1].join('.')
+    println "git.properties JSON file not found: ${inputFile.path}"
   }
 }
 
-processResources {
-  filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString])
-  filter(ReplaceTokens, tokens:[cliMajorVersion: cliMajorVersion])
-}
-
 task printVersionDetails() {
   println("fullVersion=" + detailedVersionString)
   println("cliMajorVersion=" + cliMajorVersion)
   println("version=" +  version)
 }
+
+processResources {
+  filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString])
+  filter(ReplaceTokens, tokens:[cliMajorVersion: cliMajorVersion])
+}
+

metadata-events/mxe-schemas/build.gradle

@@ -19,19 +19,21 @@ generateDataTemplate.dependsOn copyMetadataModels
 mainCopySchemas.dependsOn copyMetadataModels
 pegasus.main.generationModes = [PegasusGenerationMode.PEGASUS, PegasusGenerationMode.AVRO]
 
-task copyOriginalAvsc(type: Copy, dependsOn: generateAvroSchema) {
+task renameNamespace(type: Copy, dependsOn: generateAvroSchema) {
   from("src/mainGeneratedAvroSchema/avro")
   into file("src/renamed/avro")
-}
 
-task renameNamespace(type: Exec, dependsOn: copyOriginalAvsc) {
+  doLast {
+    project.exec {
       commandLine 'bash', './rename-namespace.sh'
+    }
+  }
 }
 
 build.dependsOn renameNamespace
 
 clean {
-  project.delete('src/main/pegasus')
-  project.delete('src/mainGeneratedAvroSchema/avro')
-  project.delete('src/renamed/avro')
+  delete 'src/main/pegasus'
+  delete 'src/mainGeneratedAvroSchema/avro'
+  delete 'src/renamed/avro'
 }

metadata-events/mxe-utils-avro/build.gradle

@@ -36,5 +36,5 @@ compileJava.dependsOn copyOriginalMXESchemas
 processResources.dependsOn copyOriginalMXESchemas
 
 clean {
-  project.delete("src/main/resources/avro")
+  delete "src/main/resources/avro"
 }

metadata-integration/java/versioning.gradle

@@ -1,60 +1,30 @@
-/**
- Applies a consistent versioning scheme to all projects using this script
-  -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4
-
-  Not providing a property will make it use git tags to mint either a version like:
-  0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories)
-
- **/
-
-
+import groovy.json.JsonSlurper
 import org.apache.tools.ant.filters.ReplaceTokens
 
+
 def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
-def snapshotVersion = false
-if (project.hasProperty("releaseVersion")) {
-  version = releaseVersion
-  detailedVersionString = releaseVersion
-} else {
-  try {
-      // apply this plugin in a try-catch block so that we can handle cases without .git directory
-      apply plugin: "com.palantir.git-version"
-      def details = versionDetails()
-      detailedVersionString = gitVersion()
-      version = details.lastTag
-      version = version.startsWith("v")? version.substring(1): version
-      def suffix = details.isCleanTag? "": "-SNAPSHOT"
-      snapshotVersion = ! details.isCleanTag
-    }
-    catch (Exception e) {
-          e.printStackTrace()
-          // last fall back
-          version = detailedVersionString
-      }
-  }
-    // trim version if it is of size 4 to size 3
-    def versionParts = version.tokenize(".")
-    if (versionParts.size() > 3) {
-      // at-least 4 part version
-      // we check if the 4th part is a .0 in which case we want to create a release
-      if ((versionParts.size() == 4) && (versionParts[3] == '0')) {
-        versionParts = versionParts[0..2]
-      }
-      version = versionParts[0..2].join('.')
-      if (versionParts.size() > 3) {
-          version = version + "-" + versionParts[3..versionParts.size()-1].join('-')
-      }
-    }
+def cliMajorVersion = "0.15.0" // base default cli major version
 
-    if (snapshotVersion) {
-        if (versionParts[versionParts.size()-1].isInteger()) {
-          version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
+task readJsonData {
+  def inputFile = file("${rootProject.buildDir}/version.json")
+
+  if (inputFile.exists()) {
+    def jsonSlurper = new JsonSlurper()
+    def data = jsonSlurper.parse(inputFile)
+
+    detailedVersionString = data.fullVersion
+    cliMajorVersion = data.cliMajorVersion
+    version = data.javaVersion
   } else {
-          // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
-          version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
-      }
+    println "JSON file not found: ${inputFile.path}"
   }
+}
 
-    processResources {
+processResources {
   filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString])
-    }
+}
+
+task printVersionDetails() {
+  println("fullVersion=" + detailedVersionString)
+  println("version=" +  version)
+}

metadata-io/build.gradle

@@ -159,5 +159,7 @@ tasks.withType(Test) {
 }
 
 clean {
-  project.delete("$projectDir/generated")
+  //TODO: Is this really needed? dont see generated folder in projectDir. It is in build, but that doest need explicit clean
+  delete "$projectDir/generated"
 }
+

metadata-jobs/mae-consumer/build.gradle

@@ -64,5 +64,5 @@ compileJava.dependsOn avroSchemaSources
 processResources.dependsOn avroSchemaSources
 
 clean {
-    project.delete("src/main/resources/avro")
+    delete "src/main/resources/avro"
 }

metadata-jobs/mce-consumer/build.gradle

@@ -58,5 +58,5 @@ compileJava.dependsOn avroSchemaSources
 processResources.dependsOn avroSchemaSources
 
 clean {
-    project.delete("src/main/resources/avro")
+    delete "src/main/resources/avro"
 }

metadata-jobs/pe-consumer/build.gradle

@@ -44,5 +44,5 @@ compileJava.dependsOn avroSchemaSources
 processResources.dependsOn avroSchemaSources
 
 clean {
-  project.delete("src/main/resources/avro")
+  delete "src/main/resources/avro"
 }

metadata-models/build.gradle

@@ -74,7 +74,6 @@ task openApiGenerate(type: GenerateSwaggerCode, dependsOn: 'generateJsonSchema')
 }
 tasks.getByName("compileJava").dependsOn(openApiGenerate)
 
-task cleanExtraDirs {
+clean {
   delete "$projectDir/src/generatedJsonSchema"
 }
-clean.finalizedBy(cleanExtraDirs)