feat(jars): better jar versioning for datahub-client, spark-lineage and protobuf (#5883)

.github/workflows/publish-datahub-jars.yml

@@ -1,40 +1,54 @@
-name: Publish Datahub Client
+name: Publish Datahub Java Jars (Client, Spark Lineage, Protobuf)
 
 on:
-  workflow_run:
+  push:
-    workflows: ["build & test"]
+    branches:
-    types:
+      - master
-      - completed
+    paths-ignore:
-
+      - "docs/**"
+      - "**.md"
   release:
-    types: [published, edited]
+    types: [published]
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
-
   check-secret:
     runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     outputs:
-      publish-enabled: ${{ steps.publish-enabled.outputs.defined }}
+      publish-enabled: ${{ steps.publish-enabled.outputs.publish }}
     steps:
       - id: publish-enabled
-        if: "${{ env.SIGNING_KEY != '' }}"
-        run: echo "::set-output name=defined::true"
         env:
           SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
-
+        run: |
+          echo "Enable publish: ${{ env.SIGNING_KEY != '' }}"
+          echo "::set-output name=publish::${{ env.SIGNING_KEY != '' }}"
+  setup:
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Compute Tag
+        id: tag
+        run: |
+          echo "GITHUB_REF: $GITHUB_REF"
+          SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
+          TAG=$(echo ${GITHUB_REF} | sed -e 's,refs/tags/v,,g')
+          echo "tag=$TAG"
+          echo "::set-output name=tag::$TAG"
   publish:
     runs-on: ubuntu-latest
-    needs: [check-secret]
+    needs: check-secret
-    if: needs.check-secret.outputs.publish-enabled == 'true'
+    if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }}
     steps:
       - uses: actions/checkout@v2
         with:
@@ -50,7 +64,8 @@ jobs:
         run: |
           git remote add upstream https://github.com/datahub-project/datahub.git
           git fetch upstream --tags --force
-      - name: publish datahub-client jar
+      - name: publish datahub-client jar snapshot
+        if: ${{ github.event_name != 'release' }}
         env:
           RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
           RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
@@ -61,9 +76,22 @@ jobs:
         run: |
           echo signingKey=$SIGNING_KEY >> gradle.properties
           ./gradlew :metadata-integration:java:datahub-client:printVersion
-          ./gradlew :metadata-integration:java:datahub-client:publishToMavenLocal
+          ./gradlew :metadata-integration:java:datahub-client:publish
-          #./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info
+      - name: release datahub-client jar
-      - name: publish datahub-spark jar
+        if: ${{ github.event_name == 'release' }}
+        env:
+          RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
+          RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
+          SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
+          SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
+          NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
+          NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+        run: |
+          echo signingKey=$SIGNING_KEY >> gradle.properties
+          ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:datahub-client:publish
+          ./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info
+      - name: publish datahub-spark-lineage snapshot jar
+        if: ${{ github.event_name != 'release' }}
         env:
           RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
           RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
@@ -74,8 +102,22 @@ jobs:
         run: |
           echo signingKey=$SIGNING_KEY >> gradle.properties
           ./gradlew :metadata-integration:java:spark-lineage:printVersion
-          ./gradlew :metadata-integration:java:spark-lineage:publishToMavenLocal
+          ./gradlew :metadata-integration:java:spark-lineage:publish
-      - name: build and publish datahub-protobuf
+      - name: release datahub-spark-lineage jar
+        if: ${{ github.event_name == 'release' }}
+        env:
+          RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
+          RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
+          SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
+          SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
+          NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
+          NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+        run: |
+          echo signingKey=$SIGNING_KEY >> gradle.properties
+          ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:spark-lineage:publish
+          ./gradlew :metadata-integration:java:spark-lineage:closeAndReleaseRepository --info
+      - name: publish datahub-protobuf snapshot jar
+        if: ${{ github.event_name != 'release' }}
         env:
           RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
           RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
@@ -85,10 +127,21 @@ jobs:
           NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
         run: |
           echo signingKey=$SIGNING_KEY >> gradle.properties
-           # Publish datahub-protobuf to maven
           ./gradlew :metadata-integration:java:datahub-protobuf:printVersion
-          ./gradlew :metadata-integration:java:datahub-protobuf:publishToMavenLocal
+          ./gradlew :metadata-integration:java:datahub-protobuf:publish
-          #./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info
+      - name: release datahub-protobuf jar
+        if: ${{ github.event_name == 'release' }}
+        env:
+          RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
+          RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
+          SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
+          SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
+          NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
+          NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+        run: |
+          echo signingKey=$SIGNING_KEY >> gradle.properties
+          ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:datahub-protobuf:publish
+          ./gradlew :metadata-integration:java:datahub-protobuf:closeAndReleaseRepository --info
       - name: build and publish datahub-ranger-plugin
         env:
           RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
@@ -102,4 +155,3 @@ jobs:
            # Publish apache ranger plugin to maven
           ./gradlew :datahub-ranger-plugin:printVersion
           ./gradlew :datahub-ranger-plugin:publishMavenJavaPublicationToMavenLocal
-          #./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info

metadata-integration/java/datahub-client/build.gradle

@@ -8,6 +8,7 @@ apply plugin: 'signing'
 apply plugin: 'io.codearte.nexus-staging'
 apply plugin: 'maven-publish'
 apply plugin: 'org.hidetake.swagger.generator'
+apply from: "../versioning.gradle"
 import org.apache.tools.ant.filters.ReplaceTokens
 
 jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation
@@ -56,54 +57,6 @@ jacocoTestReport {
     dependsOn test // tests are required to run before generating the report
 }
 
-
-def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
-def snapshotVersion = false
-if (project.hasProperty("releaseVersion")) {
-  version = releaseVersion
-  detailedVersionString = releaseVersion
-} else {
-  try {
-      // apply this plugin in a try-catch block so that we can handle cases without .git directory
-      apply plugin: "com.palantir.git-version"
-      def details = versionDetails()
-      detailedVersionString = gitVersion()
-      version = details.lastTag
-      version = version.startsWith("v")? version.substring(1): version
-      def suffix = details.isCleanTag? "": "-SNAPSHOT"
-      snapshotVersion = ! details.isCleanTag
-    }
-    catch (Exception e) {
-          e.printStackTrace()
-          // last fall back
-          version = detailedVersionString
-      }
-  }
-    // trim version if it is of size 4 to size 3
-    def versionParts = version.tokenize(".")
-    if (versionParts.size() > 3) {
-      // at-least 4 part version
-      // we check if the 4th part is a .0 in which case we want to create a release
-      if (versionParts[3] != '0') {
-        snapshotVersion = true
-      }
-      versionParts = versionParts[0..2]
-      version = versionParts[0..2].join('.')
-    }
-
-    if (snapshotVersion) {
-        if (versionParts[versionParts.size()-1].isInteger()) {
-          version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
-        } else {
-          // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
-          version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
-      }
-    }
-
-    processResources {
-      filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString])
-    }
-
 test {
   // to avoid simultaneous executions of tests when complete build is run
   mustRunAfter(":metadata-io:test")

metadata-integration/java/datahub-protobuf/build.gradle

@@ -7,7 +7,7 @@ apply plugin: 'com.github.johnrengelman.shadow'
 apply plugin: 'signing'
 apply plugin: 'io.codearte.nexus-staging'
 apply plugin: 'maven-publish'
-import org.apache.tools.ant.filters.ReplaceTokens
+apply from: '../versioning.gradle'
 
 jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation
 
@@ -73,54 +73,6 @@ configurations {
     implementation.extendsFrom provided
 }
 
-def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
-def snapshotVersion = false
-if (project.hasProperty("releaseVersion")) {
-  version = releaseVersion
-  detailedVersionString = releaseVersion
-} else {
-  try {
-      // apply this plugin in a try-catch block so that we can handle cases without .git directory
-      apply plugin: "com.palantir.git-version"
-      def details = versionDetails()
-      detailedVersionString = gitVersion()
-      version = details.lastTag
-      version = version.startsWith("v")? version.substring(1): version
-      def suffix = details.isCleanTag? "": "-SNAPSHOT"
-      snapshotVersion = ! details.isCleanTag
-    }
-    catch (Exception e) {
-          e.printStackTrace()
-          // last fall back
-          version = detailedVersionString
-      }
-  }
-    // trim version if it is of size 4 to size 3
-    def versionParts = version.tokenize(".")
-    if (versionParts.size() > 3) {
-      // at-least 4 part version
-      // we check if the 4th part is a .0 in which case we want to create a release
-      if (versionParts[3] != '0') {
-        snapshotVersion = true
-      }
-      versionParts = versionParts[0..2]
-      version = versionParts[0..2].join('.')
-    }
-
-    if (snapshotVersion) {
-        if (versionParts[versionParts.size()-1].isInteger()) {
-          version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
-        } else {
-          // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
-          version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
-      }
-    }
-
-    processResources {
-      filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString])
-    }
-
-
 shadowJar {
   zip64=true
   archiveClassifier = ''

metadata-integration/java/spark-lineage/build.gradle

@@ -7,7 +7,7 @@ apply plugin: 'signing'
 apply plugin: 'io.codearte.nexus-staging'
 apply plugin: 'maven-publish'
 apply plugin: 'jacoco'
-import org.apache.tools.ant.filters.ReplaceTokens
+apply from: '../versioning.gradle'
 
 jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation
 
@@ -21,53 +21,6 @@ configurations {
     implementation.extendsFrom provided
 }
 
-def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
-def snapshotVersion = false
-if (project.hasProperty("releaseVersion")) {
-  version = releaseVersion
-  detailedVersionString = releaseVersion
-} else {
-  try {
-      // apply this plugin in a try-catch block so that we can handle cases without .git directory
-      apply plugin: "com.palantir.git-version"
-      def details = versionDetails()
-      detailedVersionString = gitVersion()
-      version = details.lastTag
-      version = version.startsWith("v")? version.substring(1): version
-      def suffix = details.isCleanTag? "": "-SNAPSHOT"
-      snapshotVersion = ! details.isCleanTag
-    }
-    catch (Exception e) {
-          e.printStackTrace()
-          // last fall back
-          version = detailedVersionString
-      }
-  }
-    // trim version if it is of size 4 to size 3
-    def versionParts = version.tokenize(".")
-    if (versionParts.size() > 3) {
-      // at-least 4 part version
-      // we check if the 4th part is a .0 in which case we want to create a release
-      if (versionParts[3] != '0') {
-        snapshotVersion = true
-      }
-      versionParts = versionParts[0..2]
-      version = versionParts[0..2].join('.')
-    }
-
-    if (snapshotVersion) {
-        if (versionParts[versionParts.size()-1].isInteger()) {
-          version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
-        } else {
-          // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
-          version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
-      }
-    }
-
-    processResources {
-      filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString])
-    }
-
 dependencies {
 
   //Needed for tie breaking of guava version need for spark and wiremock

metadata-integration/java/versioning.gradle

@@ -0,0 +1,60 @@
+/**
+ Applies a consistent versioning scheme to all projects using this script
+  -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4
+
+  Not providing a property will make it use git tags to mint either a version like:
+  0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories)
+
+ **/
+
+
+import org.apache.tools.ant.filters.ReplaceTokens
+
+def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
+def snapshotVersion = false
+if (project.hasProperty("releaseVersion")) {
+  version = releaseVersion
+  detailedVersionString = releaseVersion
+} else {
+  try {
+      // apply this plugin in a try-catch block so that we can handle cases without .git directory
+      apply plugin: "com.palantir.git-version"
+      def details = versionDetails()
+      detailedVersionString = gitVersion()
+      version = details.lastTag
+      version = version.startsWith("v")? version.substring(1): version
+      def suffix = details.isCleanTag? "": "-SNAPSHOT"
+      snapshotVersion = ! details.isCleanTag
+    }
+    catch (Exception e) {
+          e.printStackTrace()
+          // last fall back
+          version = detailedVersionString
+      }
+  }
+    // trim version if it is of size 4 to size 3
+    def versionParts = version.tokenize(".")
+    if (versionParts.size() > 3) {
+      // at-least 4 part version
+      // we check if the 4th part is a .0 in which case we want to create a release
+      if ((versionParts.size() == 4) && (versionParts[3] == '0')) {
+        versionParts = versionParts[0..2]
+      }
+      version = versionParts[0..2].join('.')
+      if (versionParts.size() > 3) {
+          version = version + "-" + versionParts[3..versionParts.size()-1].join('-')
+      }
+    }
+
+    if (snapshotVersion) {
+        if (versionParts[versionParts.size()-1].isInteger()) {
+          version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
+        } else {
+          // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
+          version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
+      }
+    }
+
+    processResources {
+      filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString])
+    }