diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml index 8310a9ab0b..3484c618c8 100644 --- a/.github/workflows/publish-datahub-jars.yml +++ b/.github/workflows/publish-datahub-jars.yml @@ -1,40 +1,54 @@ -name: Publish Datahub Client +name: Publish Datahub Java Jars (Client, Spark Lineage, Protobuf) on: - workflow_run: - workflows: ["build & test"] - types: - - completed - + push: + branches: + - master + paths-ignore: + - "docs/**" + - "**.md" release: - types: [published, edited] + types: [published] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: - concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - check-secret: runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} outputs: - publish-enabled: ${{ steps.publish-enabled.outputs.defined }} + publish-enabled: ${{ steps.publish-enabled.outputs.publish }} steps: - id: publish-enabled - if: "${{ env.SIGNING_KEY != '' }}" - run: echo "::set-output name=defined::true" env: SIGNING_KEY: ${{ secrets.SIGNING_KEY }} - + run: | + echo "Enable publish: ${{ env.SIGNING_KEY != '' }}" + echo "::set-output name=publish::${{ env.SIGNING_KEY != '' }}" + setup: + if: startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Compute Tag + id: tag + run: | + echo "GITHUB_REF: $GITHUB_REF" + SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA") + TAG=$(echo ${GITHUB_REF} | sed -e 's,refs/tags/v,,g') + echo "tag=$TAG" + echo "::set-output name=tag::$TAG" publish: runs-on: ubuntu-latest - needs: [check-secret] - if: needs.check-secret.outputs.publish-enabled == 'true' + needs: check-secret + if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }} steps: - uses: actions/checkout@v2 with: @@ -50,7 +64,8 @@ jobs: run: | git remote add upstream https://github.com/datahub-project/datahub.git git fetch upstream --tags --force - - name: publish datahub-client jar + - name: publish datahub-client jar snapshot + if: ${{ github.event_name != 'release' }} env: RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} @@ -61,9 +76,22 @@ jobs: run: | echo signingKey=$SIGNING_KEY >> gradle.properties ./gradlew :metadata-integration:java:datahub-client:printVersion - ./gradlew :metadata-integration:java:datahub-client:publishToMavenLocal - #./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info - - name: publish datahub-spark jar + ./gradlew :metadata-integration:java:datahub-client:publish + - name: release datahub-client jar + if: ${{ github.event_name == 'release' }} + env: + RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} + RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} + SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} + SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + run: | + echo signingKey=$SIGNING_KEY >> gradle.properties + ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:datahub-client:publish + ./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info + - name: publish datahub-spark-lineage snapshot jar + if: ${{ github.event_name != 'release' }} env: RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} @@ -74,8 +102,22 @@ jobs: run: | echo signingKey=$SIGNING_KEY >> gradle.properties ./gradlew :metadata-integration:java:spark-lineage:printVersion - ./gradlew :metadata-integration:java:spark-lineage:publishToMavenLocal - - name: build and publish datahub-protobuf + ./gradlew :metadata-integration:java:spark-lineage:publish + - name: release datahub-spark-lineage jar + if: ${{ github.event_name == 'release' }} + env: + RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} + RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} + SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} + SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + run: | + echo signingKey=$SIGNING_KEY >> gradle.properties + ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:spark-lineage:publish + ./gradlew :metadata-integration:java:spark-lineage:closeAndReleaseRepository --info + - name: publish datahub-protobuf snapshot jar + if: ${{ github.event_name != 'release' }} env: RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} @@ -85,10 +127,21 @@ jobs: NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} run: | echo signingKey=$SIGNING_KEY >> gradle.properties - # Publish datahub-protobuf to maven ./gradlew :metadata-integration:java:datahub-protobuf:printVersion - ./gradlew :metadata-integration:java:datahub-protobuf:publishToMavenLocal - #./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info + ./gradlew :metadata-integration:java:datahub-protobuf:publish + - name: release datahub-protobuf jar + if: ${{ github.event_name == 'release' }} + env: + RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} + RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} + SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} + SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + run: | + echo signingKey=$SIGNING_KEY >> gradle.properties + ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:datahub-protobuf:publish + ./gradlew :metadata-integration:java:datahub-protobuf:closeAndReleaseRepository --info - name: build and publish datahub-ranger-plugin env: RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} @@ -102,4 +155,3 @@ jobs: # Publish apache ranger plugin to maven ./gradlew :datahub-ranger-plugin:printVersion ./gradlew :datahub-ranger-plugin:publishMavenJavaPublicationToMavenLocal - #./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index 78ba4e3723..8711c530f7 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -8,6 +8,7 @@ apply plugin: 'signing' apply plugin: 'io.codearte.nexus-staging' apply plugin: 'maven-publish' apply plugin: 'org.hidetake.swagger.generator' +apply from: "../versioning.gradle" import org.apache.tools.ant.filters.ReplaceTokens jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation @@ -56,54 +57,6 @@ jacocoTestReport { dependsOn test // tests are required to run before generating the report } - -def detailedVersionString = "0.0.0-unknown-SNAPSHOT" -def snapshotVersion = false -if (project.hasProperty("releaseVersion")) { - version = releaseVersion - detailedVersionString = releaseVersion -} else { - try { - // apply this plugin in a try-catch block so that we can handle cases without .git directory - apply plugin: "com.palantir.git-version" - def details = versionDetails() - detailedVersionString = gitVersion() - version = details.lastTag - version = version.startsWith("v")? version.substring(1): version - def suffix = details.isCleanTag? "": "-SNAPSHOT" - snapshotVersion = ! details.isCleanTag - } - catch (Exception e) { - e.printStackTrace() - // last fall back - version = detailedVersionString - } - } - // trim version if it is of size 4 to size 3 - def versionParts = version.tokenize(".") - if (versionParts.size() > 3) { - // at-least 4 part version - // we check if the 4th part is a .0 in which case we want to create a release - if (versionParts[3] != '0') { - snapshotVersion = true - } - versionParts = versionParts[0..2] - version = versionParts[0..2].join('.') - } - - if (snapshotVersion) { - if (versionParts[versionParts.size()-1].isInteger()) { - version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT" - } else { - // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version - version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' - } - } - - processResources { - filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString]) - } - test { // to avoid simultaneous executions of tests when complete build is run mustRunAfter(":metadata-io:test") diff --git a/metadata-integration/java/datahub-protobuf/build.gradle b/metadata-integration/java/datahub-protobuf/build.gradle index d734509e72..6dd21bb706 100644 --- a/metadata-integration/java/datahub-protobuf/build.gradle +++ b/metadata-integration/java/datahub-protobuf/build.gradle @@ -7,7 +7,7 @@ apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'signing' apply plugin: 'io.codearte.nexus-staging' apply plugin: 'maven-publish' -import org.apache.tools.ant.filters.ReplaceTokens +apply from: '../versioning.gradle' jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation @@ -73,54 +73,6 @@ configurations { implementation.extendsFrom provided } -def detailedVersionString = "0.0.0-unknown-SNAPSHOT" -def snapshotVersion = false -if (project.hasProperty("releaseVersion")) { - version = releaseVersion - detailedVersionString = releaseVersion -} else { - try { - // apply this plugin in a try-catch block so that we can handle cases without .git directory - apply plugin: "com.palantir.git-version" - def details = versionDetails() - detailedVersionString = gitVersion() - version = details.lastTag - version = version.startsWith("v")? version.substring(1): version - def suffix = details.isCleanTag? "": "-SNAPSHOT" - snapshotVersion = ! details.isCleanTag - } - catch (Exception e) { - e.printStackTrace() - // last fall back - version = detailedVersionString - } - } - // trim version if it is of size 4 to size 3 - def versionParts = version.tokenize(".") - if (versionParts.size() > 3) { - // at-least 4 part version - // we check if the 4th part is a .0 in which case we want to create a release - if (versionParts[3] != '0') { - snapshotVersion = true - } - versionParts = versionParts[0..2] - version = versionParts[0..2].join('.') - } - - if (snapshotVersion) { - if (versionParts[versionParts.size()-1].isInteger()) { - version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT" - } else { - // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version - version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' - } - } - - processResources { - filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString]) - } - - shadowJar { zip64=true archiveClassifier = '' diff --git a/metadata-integration/java/spark-lineage/build.gradle b/metadata-integration/java/spark-lineage/build.gradle index f04878d5bf..18fc84bda0 100644 --- a/metadata-integration/java/spark-lineage/build.gradle +++ b/metadata-integration/java/spark-lineage/build.gradle @@ -7,7 +7,7 @@ apply plugin: 'signing' apply plugin: 'io.codearte.nexus-staging' apply plugin: 'maven-publish' apply plugin: 'jacoco' -import org.apache.tools.ant.filters.ReplaceTokens +apply from: '../versioning.gradle' jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation @@ -21,53 +21,6 @@ configurations { implementation.extendsFrom provided } -def detailedVersionString = "0.0.0-unknown-SNAPSHOT" -def snapshotVersion = false -if (project.hasProperty("releaseVersion")) { - version = releaseVersion - detailedVersionString = releaseVersion -} else { - try { - // apply this plugin in a try-catch block so that we can handle cases without .git directory - apply plugin: "com.palantir.git-version" - def details = versionDetails() - detailedVersionString = gitVersion() - version = details.lastTag - version = version.startsWith("v")? version.substring(1): version - def suffix = details.isCleanTag? "": "-SNAPSHOT" - snapshotVersion = ! details.isCleanTag - } - catch (Exception e) { - e.printStackTrace() - // last fall back - version = detailedVersionString - } - } - // trim version if it is of size 4 to size 3 - def versionParts = version.tokenize(".") - if (versionParts.size() > 3) { - // at-least 4 part version - // we check if the 4th part is a .0 in which case we want to create a release - if (versionParts[3] != '0') { - snapshotVersion = true - } - versionParts = versionParts[0..2] - version = versionParts[0..2].join('.') - } - - if (snapshotVersion) { - if (versionParts[versionParts.size()-1].isInteger()) { - version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT" - } else { - // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version - version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' - } - } - - processResources { - filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString]) - } - dependencies { //Needed for tie breaking of guava version need for spark and wiremock diff --git a/metadata-integration/java/versioning.gradle b/metadata-integration/java/versioning.gradle new file mode 100644 index 0000000000..baca5641f6 --- /dev/null +++ b/metadata-integration/java/versioning.gradle @@ -0,0 +1,60 @@ +/** + Applies a consistent versioning scheme to all projects using this script + -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4 + + Not providing a property will make it use git tags to mint either a version like: + 0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories) + + **/ + + +import org.apache.tools.ant.filters.ReplaceTokens + +def detailedVersionString = "0.0.0-unknown-SNAPSHOT" +def snapshotVersion = false +if (project.hasProperty("releaseVersion")) { + version = releaseVersion + detailedVersionString = releaseVersion +} else { + try { + // apply this plugin in a try-catch block so that we can handle cases without .git directory + apply plugin: "com.palantir.git-version" + def details = versionDetails() + detailedVersionString = gitVersion() + version = details.lastTag + version = version.startsWith("v")? version.substring(1): version + def suffix = details.isCleanTag? "": "-SNAPSHOT" + snapshotVersion = ! details.isCleanTag + } + catch (Exception e) { + e.printStackTrace() + // last fall back + version = detailedVersionString + } + } + // trim version if it is of size 4 to size 3 + def versionParts = version.tokenize(".") + if (versionParts.size() > 3) { + // at-least 4 part version + // we check if the 4th part is a .0 in which case we want to create a release + if ((versionParts.size() == 4) && (versionParts[3] == '0')) { + versionParts = versionParts[0..2] + } + version = versionParts[0..2].join('.') + if (versionParts.size() > 3) { + version = version + "-" + versionParts[3..versionParts.size()-1].join('-') + } + } + + if (snapshotVersion) { + if (versionParts[versionParts.size()-1].isInteger()) { + version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT" + } else { + // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version + version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' + } + } + + processResources { + filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString]) + }