feat(datahub-protobuf): add support for shadow jar, publish (#5882)

.github/workflows/build-and-test.yml

@@ -38,7 +38,7 @@ jobs:
         # datahub-frontend:unzipAssets depends on datahub-web-react:yarnBuild but gradle does not know about it
         run: |
           ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x datahub-web-react:yarnBuild -x datahub-frontend:unzipAssets -x :metadata-io:test -x :metadata-integration:java:datahub-protobuf:build
-          ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-integration:java:datahub-protobuf:build
+          ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test
       - uses: actions/upload-artifact@v2
         if: always()
         with:

.github/workflows/check-datahub-jars.yml

@@ -45,3 +45,7 @@ jobs:
         run: |
           ./gradlew :metadata-integration:java:spark-lineage:build
           ./gradlew :metadata-integration:java:spark-lineage:javadoc
+      - name: check datahub-protobuf jar
+        run: |
+          ./gradlew :metadata-integration:java:datahub-protobuf:build
+          ./gradlew :metadata-integration:java:datahub-protobuf:javadoc

.github/workflows/publish-datahub-jars.yml

@@ -75,6 +75,30 @@ jobs:
           echo signingKey=$SIGNING_KEY >> gradle.properties
           ./gradlew :metadata-integration:java:spark-lineage:printVersion
           ./gradlew :metadata-integration:java:spark-lineage:publishToMavenLocal
+      - name: build and publish datahub-protobuf
+        env:
+          RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
+          RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
+          SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
+          SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
+          NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
+          NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+        run: |
+          echo signingKey=$SIGNING_KEY >> gradle.properties
+           # Publish datahub-protobuf to maven
+          ./gradlew :metadata-integration:java:datahub-protobuf:printVersion
+          ./gradlew :metadata-integration:java:datahub-protobuf:publishToMavenLocal
+          #./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info
+      - name: build and publish datahub-ranger-plugin
+        env:
+          RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }}
+          RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }}
+          SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }}
+          SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
+          NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
+          NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+        run: |
+          echo signingKey=$SIGNING_KEY >> gradle.properties
            # Publish apache ranger plugin to maven
           ./gradlew :datahub-ranger-plugin:printVersion
           ./gradlew :datahub-ranger-plugin:publishMavenJavaPublicationToMavenLocal

build.gradle

@@ -230,6 +230,11 @@ subprojects {
       }
     }
   } else {
+    tasks.withType(Javadoc).configureEach {
+    javadocTool = javaToolchains.javadocToolFor {
+        languageVersion = JavaLanguageVersion.of(11)
+      }
+    }
     tasks.withType(JavaCompile).configureEach {
       javaCompiler = javaToolchains.compilerFor {
         languageVersion = JavaLanguageVersion.of(11)

metadata-integration/java/datahub-protobuf-example/build.gradle

@@ -5,10 +5,12 @@ plugins {
 
 repositories {
     mavenCentral()
+    mavenLocal()
 }
 
 ext {
     protobuf_version = '3.19.3'
+    datahub_protobuf_version = '0.8.45-SNAPSHOT'
 }
 
 configurations {
@@ -18,15 +20,7 @@ configurations {
 dependencies {
     // compilation
     implementation "com.google.protobuf:protobuf-java:$protobuf_version"
-
-    datahub files('libs/datahub-protobuf.jar')
-
-    // transitive deps
-    datahub 'io.acryl:datahub-client:0.8.+'
-    datahub "com.google.protobuf:protobuf-java:$protobuf_version"
-    datahub 'org.jgrapht:jgrapht-core:1.5.1'
-    datahub 'com.google.guava:guava:27.0.1-jre'
-    datahub 'com.google.code.gson:gson:2.8.6'
+    datahub "io.acryl:datahub-protobuf:$datahub_protobuf_version"
 }
 
 sourceSets {

metadata-integration/java/datahub-protobuf/README.md

@@ -523,13 +523,36 @@ message Message {
 }
 ```
 
-## Gradle Integration
+## Installation
+
+Follow the specific instructions for your build system to declare a dependency on the appropriate version of the package.
+
+**_Note_**: Check the [Maven repository](https://mvnrepository.com/artifact/io.acryl/datahub-protobuf) for the latest version of the package before following the instructions below.
+
+### Gradle
+Add the following to your build.gradle.
+```gradle
+implementation 'io.acryl:datahub-protobuf:__version__'
+```
+### Maven
+Add the following to your `pom.xml`.
+```xml
+<!-- https://mvnrepository.com/artifact/io.acryl/datahub-protobuf -->
+<dependency>
+    <groupId>io.acryl</groupId>
+    <artifactId>datahub-protobuf</artifactId>
+    <!-- replace __version__ with the latest version number -->
+    <version>__version__</version>
+</dependency>
+```
+
+## Example Application
 
 An example application is included which works with the `protobuf-gradle-plugin`, see the standalone [example project](../datahub-protobuf-example).
 
 ### Usage
 
-Using the 
+Using the example application:
 
 ```shell
 export DATAHUB_API=...

metadata-integration/java/datahub-protobuf/build.gradle

@@ -3,6 +3,13 @@ plugins {
 }
 apply plugin: 'java'
 apply plugin: 'jacoco'
+apply plugin: 'com.github.johnrengelman.shadow'
+apply plugin: 'signing'
+apply plugin: 'io.codearte.nexus-staging'
+apply plugin: 'maven-publish'
+import org.apache.tools.ant.filters.ReplaceTokens
+
+jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation
 
 afterEvaluate {
     if (project.plugins.hasPlugin('java')) {
@@ -53,3 +60,189 @@ test {
     useJUnit()
     finalizedBy jacocoTestReport
 }
+
+
+task checkShadowJar(type: Exec) {
+   commandLine 'sh', '-c', 'scripts/check_jar.sh'
+}
+
+//mark implementation dependencies which needs to excluded along with transitive dependencies from shadowjar
+//functionality is exactly same as "implementation"
+configurations {
+    provided
+    implementation.extendsFrom provided
+}
+
+def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
+def snapshotVersion = false
+if (project.hasProperty("releaseVersion")) {
+  version = releaseVersion
+  detailedVersionString = releaseVersion
+} else {
+  try {
+      // apply this plugin in a try-catch block so that we can handle cases without .git directory
+      apply plugin: "com.palantir.git-version"
+      def details = versionDetails()
+      detailedVersionString = gitVersion()
+      version = details.lastTag
+      version = version.startsWith("v")? version.substring(1): version
+      def suffix = details.isCleanTag? "": "-SNAPSHOT"
+      snapshotVersion = ! details.isCleanTag
+    }
+    catch (Exception e) {
+          e.printStackTrace()
+          // last fall back
+          version = detailedVersionString
+      }
+  }
+    // trim version if it is of size 4 to size 3
+    def versionParts = version.tokenize(".")
+    if (versionParts.size() > 3) {
+      // at-least 4 part version
+      // we check if the 4th part is a .0 in which case we want to create a release
+      if (versionParts[3] != '0') {
+        snapshotVersion = true
+      }
+      versionParts = versionParts[0..2]
+      version = versionParts[0..2].join('.')
+    }
+
+    if (snapshotVersion) {
+        if (versionParts[versionParts.size()-1].isInteger()) {
+          version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
+        } else {
+          // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
+          version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
+      }
+    }
+
+    processResources {
+      filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString])
+    }
+
+
+shadowJar {
+  zip64=true
+  archiveClassifier = ''
+  def exclude_modules = project
+            .configurations
+            .provided
+            .resolvedConfiguration
+            .getLenientConfiguration()
+            .getAllModuleDependencies()
+            .collect {
+                it.name
+            }
+  dependencies {
+
+    exclude(dependency {
+            exclude_modules.contains(it.name)
+        })
+
+  }
+  mergeServiceFiles()
+  // we relocate namespaces manually, because we want to know exactly which libs we are exposing and why
+  // we can move to automatic relocation using ConfigureShadowRelocation after we get to a good place on these first
+    relocate 'com.fasterxml.jackson', 'datahub.shaded.jackson'
+    relocate 'net.jcip.annotations', 'datahub.shaded.annotations'
+    relocate 'javassist', 'datahub.shaded.javassist'
+    relocate 'edu.umd.cs.findbugs', 'datahub.shaded.findbugs'
+    relocate 'org.antlr', 'datahub.shaded.org.antlr'
+     relocate 'antlr', 'datahub.shaded.antlr'
+    relocate 'org.apache.commons', 'datahub.shaded.org.apache.commons'
+    relocate 'org.apache.http', 'datahub.shaded.org.apache.http'
+    relocate 'org.reflections', 'datahub.shaded.org.reflections'
+    relocate 'st4hidden', 'datahub.shaded.st4hidden'
+    relocate 'org.stringtemplate', 'datahub.shaded.org.stringtemplate'
+    relocate 'org.abego.treelayout', 'datahub.shaded.treelayout'
+    relocate 'org.slf4j', 'datahub.shaded.slf4j'
+    relocate 'com.github.benmanes.caffeine', 'datahub.shaded.com.github.benmanes.caffeine'
+    relocate 'org.checkerframework', 'datahub.shaded.org.checkerframework'
+    relocate 'org.jgrapht', 'datahub.shaded.org.jgrapht'
+    relocate 'org.jheaps', 'datahub.shaded.org.jheaps'
+ 
+  finalizedBy checkShadowJar
+}
+
+checkShadowJar {
+   dependsOn shadowJar
+}
+
+assemble {
+    dependsOn shadowJar
+}
+
+task sourcesJar(type: Jar) {
+     archiveClassifier = 'sources'
+     from sourceSets.main.allSource
+}
+
+task javadocJar(type: Jar) {
+    archiveClassifier = 'javadoc'
+    from javadoc
+}
+
+
+publishing {
+  publications {
+    shadow(MavenPublication) {
+      publication -> project.shadow.component(publication)
+      pom {
+        name = 'Datahub Protobuf'
+        group = 'io.acryl'
+        artifactId = 'datahub-protobuf'
+        description = 'DataHub integration with Protobuf schemas for metadata'
+        url = 'https://datahubproject.io'
+        artifacts = [ shadowJar, javadocJar, sourcesJar ]
+
+        scm {
+          connection = 'scm:git:git://github.com/datahub-project/datahub.git'
+          developerConnection = 'scm:git:ssh://github.com:datahub-project/datahub.git'
+          url = 'https://github.com/datahub-project/datahub.git'
+        }
+
+        licenses {
+          license {
+            name = 'The Apache License, Version 2.0'
+            url = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+          }
+        }
+
+        developers {
+          developer {
+            id = 'datahub'
+            name = 'Datahub'
+            email = 'datahub@acryl.io'
+          }
+        }
+      }
+    }
+  }
+
+  repositories {
+    maven {
+      def releasesRepoUrl = "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/"
+      def snapshotsRepoUrl = "https://s01.oss.sonatype.org/content/repositories/snapshots/"
+      def ossrhUsername = System.getenv('RELEASE_USERNAME')
+      def ossrhPassword = System.getenv('RELEASE_PASSWORD')
+      credentials {
+        username ossrhUsername
+        password ossrhPassword
+      }
+      url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
+    }
+  }
+}
+
+signing {
+    def signingKey = findProperty("signingKey")
+    def signingPassword = System.getenv("SIGNING_PASSWORD")
+    useInMemoryPgpKeys(signingKey, signingPassword)
+    sign publishing.publications.shadow
+}
+
+nexusStaging {
+    serverUrl = "https://s01.oss.sonatype.org/service/local/" //required only for projects registered in Sonatype after 2021-02-24
+    username = System.getenv("NEXUS_USERNAME")
+    password = System.getenv("NEXUS_PASSWORD")
+}

metadata-integration/java/datahub-protobuf/scripts/check_jar.sh

@@ -0,0 +1,48 @@
+# This script checks the shadow jar to ensure that we only have allowed classes being exposed through the jar
+jarFiles=$(find build/libs -name "datahub-protobuf*.jar" | grep -v sources | grep -v javadoc)
+for jarFile in ${jarFiles}; do
+jar -tvf $jarFile |\
+      grep -v "datahub/shaded" |\
+      grep -v "google/" |\
+      grep -v "google/protobuf" |\
+      grep -v "protobuf/" |\
+      grep -v "mozilla/" |\
+      grep -v "META-INF" |\
+      grep -v "com/linkedin" |\
+      grep -v "com/datahub" |\
+      grep -v "datahub" |\
+      grep -v "entity-registry" |\
+      grep -v "pegasus/" |\
+      grep -v "legacyPegasusSchemas/" |\
+      grep -v " com/$" |\
+      grep -v " org/$" |\
+      grep -v " io/$" |\
+      grep -v "git.properties" |\
+      grep -v "org/springframework" |\
+      grep -v "org/aopalliance" |\
+      grep -v "javax/" |\
+      grep -v "io/swagger" |\
+      grep -v "JavaSpring" |\
+      grep -v "java-header-style.xml" |\
+      grep -v "xml-header-style.xml" |\
+      grep -v "license.header" |\
+      grep -v "module-info.class" |\
+      grep -v "com/google/" |\
+      grep -v "org/codehaus/" |\
+      grep -v "client.properties" |\
+      grep -v "kafka" |\
+      grep -v "win/" |\
+      grep -v "include/" |\
+      grep -v "linux/" |\
+      grep -v "darwin" |\
+      grep -v "MetadataChangeProposal.avsc" |\
+      grep -v "aix"
+
+if [ $? -ne 0 ]; then
+  echo "✅ No unexpected class paths found in ${jarFile}"
+else
+  echo "💥 Found unexpected class paths in ${jarFile}"
+  exit 1
+fi
+done
+exit 0

metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/VisitContext.java

@@ -54,4 +54,8 @@ public class VisitContext {
                 .reduce(Double::sum)
                 .orElse(0);
     }
+
+    public static class VisitContextBuilder {
+
+    };
 }