diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index b8224d1f75..68f219ddc7 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -38,7 +38,7 @@ jobs: # datahub-frontend:unzipAssets depends on datahub-web-react:yarnBuild but gradle does not know about it run: | ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x datahub-web-react:yarnBuild -x datahub-frontend:unzipAssets -x :metadata-io:test -x :metadata-integration:java:datahub-protobuf:build - ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-integration:java:datahub-protobuf:build + ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test - uses: actions/upload-artifact@v2 if: always() with: diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml index e3e1925b4b..a1d7a1f47d 100644 --- a/.github/workflows/check-datahub-jars.yml +++ b/.github/workflows/check-datahub-jars.yml @@ -45,3 +45,7 @@ jobs: run: | ./gradlew :metadata-integration:java:spark-lineage:build ./gradlew :metadata-integration:java:spark-lineage:javadoc + - name: check datahub-protobuf jar + run: | + ./gradlew :metadata-integration:java:datahub-protobuf:build + ./gradlew :metadata-integration:java:datahub-protobuf:javadoc diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml index 3d77cb4dc8..8310a9ab0b 100644 --- a/.github/workflows/publish-datahub-jars.yml +++ b/.github/workflows/publish-datahub-jars.yml @@ -75,7 +75,31 @@ jobs: echo signingKey=$SIGNING_KEY >> gradle.properties ./gradlew :metadata-integration:java:spark-lineage:printVersion ./gradlew :metadata-integration:java:spark-lineage:publishToMavenLocal - # Publish apache ranger plugin to maven + - name: build and publish datahub-protobuf + env: + RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} + RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} + SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} + SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + run: | + echo signingKey=$SIGNING_KEY >> gradle.properties + # Publish datahub-protobuf to maven + ./gradlew :metadata-integration:java:datahub-protobuf:printVersion + ./gradlew :metadata-integration:java:datahub-protobuf:publishToMavenLocal + #./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info + - name: build and publish datahub-ranger-plugin + env: + RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} + RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} + SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} + SIGNING_KEY: ${{ secrets.SIGNING_KEY }} + NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + run: | + echo signingKey=$SIGNING_KEY >> gradle.properties + # Publish apache ranger plugin to maven ./gradlew :datahub-ranger-plugin:printVersion ./gradlew :datahub-ranger-plugin:publishMavenJavaPublicationToMavenLocal #./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info diff --git a/build.gradle b/build.gradle index 6914bd1f36..acf926ca95 100644 --- a/build.gradle +++ b/build.gradle @@ -230,6 +230,11 @@ subprojects { } } } else { + tasks.withType(Javadoc).configureEach { + javadocTool = javaToolchains.javadocToolFor { + languageVersion = JavaLanguageVersion.of(11) + } + } tasks.withType(JavaCompile).configureEach { javaCompiler = javaToolchains.compilerFor { languageVersion = JavaLanguageVersion.of(11) diff --git a/metadata-integration/java/datahub-protobuf-example/build.gradle b/metadata-integration/java/datahub-protobuf-example/build.gradle index 38ce29c455..10db3a4620 100644 --- a/metadata-integration/java/datahub-protobuf-example/build.gradle +++ b/metadata-integration/java/datahub-protobuf-example/build.gradle @@ -5,10 +5,12 @@ plugins { repositories { mavenCentral() + mavenLocal() } ext { protobuf_version = '3.19.3' + datahub_protobuf_version = '0.8.45-SNAPSHOT' } configurations { @@ -18,15 +20,7 @@ configurations { dependencies { // compilation implementation "com.google.protobuf:protobuf-java:$protobuf_version" - - datahub files('libs/datahub-protobuf.jar') - - // transitive deps - datahub 'io.acryl:datahub-client:0.8.+' - datahub "com.google.protobuf:protobuf-java:$protobuf_version" - datahub 'org.jgrapht:jgrapht-core:1.5.1' - datahub 'com.google.guava:guava:27.0.1-jre' - datahub 'com.google.code.gson:gson:2.8.6' + datahub "io.acryl:datahub-protobuf:$datahub_protobuf_version" } sourceSets { diff --git a/metadata-integration/java/datahub-protobuf-example/libs/datahub-protobuf.jar b/metadata-integration/java/datahub-protobuf-example/libs/datahub-protobuf.jar deleted file mode 100644 index 53486ecb61..0000000000 Binary files a/metadata-integration/java/datahub-protobuf-example/libs/datahub-protobuf.jar and /dev/null differ diff --git a/metadata-integration/java/datahub-protobuf/README.md b/metadata-integration/java/datahub-protobuf/README.md index e7e4bd23a0..24ed2c3d99 100644 --- a/metadata-integration/java/datahub-protobuf/README.md +++ b/metadata-integration/java/datahub-protobuf/README.md @@ -523,13 +523,36 @@ message Message { } ``` -## Gradle Integration +## Installation + +Follow the specific instructions for your build system to declare a dependency on the appropriate version of the package. + +**_Note_**: Check the [Maven repository](https://mvnrepository.com/artifact/io.acryl/datahub-protobuf) for the latest version of the package before following the instructions below. + +### Gradle +Add the following to your build.gradle. +```gradle +implementation 'io.acryl:datahub-protobuf:__version__' +``` +### Maven +Add the following to your `pom.xml`. +```xml + + + io.acryl + datahub-protobuf + + __version__ + +``` + +## Example Application An example application is included which works with the `protobuf-gradle-plugin`, see the standalone [example project](../datahub-protobuf-example). ### Usage -Using the +Using the example application: ```shell export DATAHUB_API=... diff --git a/metadata-integration/java/datahub-protobuf/build.gradle b/metadata-integration/java/datahub-protobuf/build.gradle index 21f4ecb1c7..d734509e72 100644 --- a/metadata-integration/java/datahub-protobuf/build.gradle +++ b/metadata-integration/java/datahub-protobuf/build.gradle @@ -3,6 +3,13 @@ plugins { } apply plugin: 'java' apply plugin: 'jacoco' +apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'signing' +apply plugin: 'io.codearte.nexus-staging' +apply plugin: 'maven-publish' +import org.apache.tools.ant.filters.ReplaceTokens + +jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation afterEvaluate { if (project.plugins.hasPlugin('java')) { @@ -52,4 +59,190 @@ jacocoTestReport { test { useJUnit() finalizedBy jacocoTestReport -} \ No newline at end of file +} + + +task checkShadowJar(type: Exec) { + commandLine 'sh', '-c', 'scripts/check_jar.sh' +} + +//mark implementation dependencies which needs to excluded along with transitive dependencies from shadowjar +//functionality is exactly same as "implementation" +configurations { + provided + implementation.extendsFrom provided +} + +def detailedVersionString = "0.0.0-unknown-SNAPSHOT" +def snapshotVersion = false +if (project.hasProperty("releaseVersion")) { + version = releaseVersion + detailedVersionString = releaseVersion +} else { + try { + // apply this plugin in a try-catch block so that we can handle cases without .git directory + apply plugin: "com.palantir.git-version" + def details = versionDetails() + detailedVersionString = gitVersion() + version = details.lastTag + version = version.startsWith("v")? version.substring(1): version + def suffix = details.isCleanTag? "": "-SNAPSHOT" + snapshotVersion = ! details.isCleanTag + } + catch (Exception e) { + e.printStackTrace() + // last fall back + version = detailedVersionString + } + } + // trim version if it is of size 4 to size 3 + def versionParts = version.tokenize(".") + if (versionParts.size() > 3) { + // at-least 4 part version + // we check if the 4th part is a .0 in which case we want to create a release + if (versionParts[3] != '0') { + snapshotVersion = true + } + versionParts = versionParts[0..2] + version = versionParts[0..2].join('.') + } + + if (snapshotVersion) { + if (versionParts[versionParts.size()-1].isInteger()) { + version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT" + } else { + // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version + version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' + } + } + + processResources { + filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString]) + } + + +shadowJar { + zip64=true + archiveClassifier = '' + def exclude_modules = project + .configurations + .provided + .resolvedConfiguration + .getLenientConfiguration() + .getAllModuleDependencies() + .collect { + it.name + } + dependencies { + + exclude(dependency { + exclude_modules.contains(it.name) + }) + + } + mergeServiceFiles() + // we relocate namespaces manually, because we want to know exactly which libs we are exposing and why + // we can move to automatic relocation using ConfigureShadowRelocation after we get to a good place on these first + relocate 'com.fasterxml.jackson', 'datahub.shaded.jackson' + relocate 'net.jcip.annotations', 'datahub.shaded.annotations' + relocate 'javassist', 'datahub.shaded.javassist' + relocate 'edu.umd.cs.findbugs', 'datahub.shaded.findbugs' + relocate 'org.antlr', 'datahub.shaded.org.antlr' + relocate 'antlr', 'datahub.shaded.antlr' + relocate 'org.apache.commons', 'datahub.shaded.org.apache.commons' + relocate 'org.apache.http', 'datahub.shaded.org.apache.http' + relocate 'org.reflections', 'datahub.shaded.org.reflections' + relocate 'st4hidden', 'datahub.shaded.st4hidden' + relocate 'org.stringtemplate', 'datahub.shaded.org.stringtemplate' + relocate 'org.abego.treelayout', 'datahub.shaded.treelayout' + relocate 'org.slf4j', 'datahub.shaded.slf4j' + relocate 'com.github.benmanes.caffeine', 'datahub.shaded.com.github.benmanes.caffeine' + relocate 'org.checkerframework', 'datahub.shaded.org.checkerframework' + relocate 'org.jgrapht', 'datahub.shaded.org.jgrapht' + relocate 'org.jheaps', 'datahub.shaded.org.jheaps' + + finalizedBy checkShadowJar +} + +checkShadowJar { + dependsOn shadowJar +} + +assemble { + dependsOn shadowJar +} + +task sourcesJar(type: Jar) { + archiveClassifier = 'sources' + from sourceSets.main.allSource +} + +task javadocJar(type: Jar) { + archiveClassifier = 'javadoc' + from javadoc +} + + +publishing { + publications { + shadow(MavenPublication) { + publication -> project.shadow.component(publication) + pom { + name = 'Datahub Protobuf' + group = 'io.acryl' + artifactId = 'datahub-protobuf' + description = 'DataHub integration with Protobuf schemas for metadata' + url = 'https://datahubproject.io' + artifacts = [ shadowJar, javadocJar, sourcesJar ] + + scm { + connection = 'scm:git:git://github.com/datahub-project/datahub.git' + developerConnection = 'scm:git:ssh://github.com:datahub-project/datahub.git' + url = 'https://github.com/datahub-project/datahub.git' + } + + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + + developers { + developer { + id = 'datahub' + name = 'Datahub' + email = 'datahub@acryl.io' + } + } + } + } + } + + repositories { + maven { + def releasesRepoUrl = "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/" + def snapshotsRepoUrl = "https://s01.oss.sonatype.org/content/repositories/snapshots/" + def ossrhUsername = System.getenv('RELEASE_USERNAME') + def ossrhPassword = System.getenv('RELEASE_PASSWORD') + credentials { + username ossrhUsername + password ossrhPassword + } + url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl + } + } +} + +signing { + def signingKey = findProperty("signingKey") + def signingPassword = System.getenv("SIGNING_PASSWORD") + useInMemoryPgpKeys(signingKey, signingPassword) + sign publishing.publications.shadow +} + +nexusStaging { + serverUrl = "https://s01.oss.sonatype.org/service/local/" //required only for projects registered in Sonatype after 2021-02-24 + username = System.getenv("NEXUS_USERNAME") + password = System.getenv("NEXUS_PASSWORD") +} diff --git a/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh b/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh new file mode 100755 index 0000000000..e04ba04fcf --- /dev/null +++ b/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh @@ -0,0 +1,48 @@ +# This script checks the shadow jar to ensure that we only have allowed classes being exposed through the jar +jarFiles=$(find build/libs -name "datahub-protobuf*.jar" | grep -v sources | grep -v javadoc) +for jarFile in ${jarFiles}; do +jar -tvf $jarFile |\ + grep -v "datahub/shaded" |\ + grep -v "google/" |\ + grep -v "google/protobuf" |\ + grep -v "protobuf/" |\ + grep -v "mozilla/" |\ + grep -v "META-INF" |\ + grep -v "com/linkedin" |\ + grep -v "com/datahub" |\ + grep -v "datahub" |\ + grep -v "entity-registry" |\ + grep -v "pegasus/" |\ + grep -v "legacyPegasusSchemas/" |\ + grep -v " com/$" |\ + grep -v " org/$" |\ + grep -v " io/$" |\ + grep -v "git.properties" |\ + grep -v "org/springframework" |\ + grep -v "org/aopalliance" |\ + grep -v "javax/" |\ + grep -v "io/swagger" |\ + grep -v "JavaSpring" |\ + grep -v "java-header-style.xml" |\ + grep -v "xml-header-style.xml" |\ + grep -v "license.header" |\ + grep -v "module-info.class" |\ + grep -v "com/google/" |\ + grep -v "org/codehaus/" |\ + grep -v "client.properties" |\ + grep -v "kafka" |\ + grep -v "win/" |\ + grep -v "include/" |\ + grep -v "linux/" |\ + grep -v "darwin" |\ + grep -v "MetadataChangeProposal.avsc" |\ + grep -v "aix" + +if [ $? -ne 0 ]; then + echo "✅ No unexpected class paths found in ${jarFile}" +else + echo "💥 Found unexpected class paths in ${jarFile}" + exit 1 +fi +done +exit 0 diff --git a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/VisitContext.java b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/VisitContext.java index 57e3d61b38..51c92332d9 100644 --- a/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/VisitContext.java +++ b/metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/visitors/VisitContext.java @@ -54,4 +54,8 @@ public class VisitContext { .reduce(Double::sum) .orElse(0); } + + public static class VisitContextBuilder { + + }; }