fix(logging): Remove lombok as source of slf4j-api, convert to compileOnly where possible (#6616)

build.gradle

@@ -10,6 +10,10 @@ buildscript {
   ext.testContainersVersion = '1.17.4'
   ext.jacksonVersion = '2.13.4'
   ext.jettyVersion = '9.4.46.v20220331'
+  ext.log4jVersion = '2.19.0'
+  ext.slf4jVersion = '1.7.32'
+  ext.logbackClassic = '1.2.11'
+
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
   dependencies {
@@ -112,9 +116,11 @@ project.ext.externalDependency = [
     'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1',
     'kafkaAvroSerializer': 'io.confluent:kafka-avro-serializer:5.1.4',
     'kafkaClients': 'org.apache.kafka:kafka-clients:2.3.0',
-    'logbackClassic': 'ch.qos.logback:logback-classic:1.2.9',
+    'logbackClassic': "ch.qos.logback:logback-classic:$logbackClassic",
-    'log4jCore': 'org.apache.logging.log4j:log4j-core:2.19.0',
+    'slf4jApi': "org.slf4j:slf4j-api:$slf4jVersion",
-    'log4jApi': 'org.apache.logging.log4j:log4j-api:2.19.0',
+    'log4jCore': "org.apache.logging.log4j:log4j-core:$log4jVersion",
+    'log4jApi': "org.apache.logging.log4j:log4j-api:$log4jVersion",
+    'log4j12Api': "org.slf4j:log4j-over-slf4j:$slf4jVersion",
     'lombok': 'org.projectlombok:lombok:1.18.12',
     'mariadbConnector': 'org.mariadb.jdbc:mariadb-java-client:2.6.0',
     'mavenArtifact': "org.apache.maven:maven-artifact:$mavenVersion",
@@ -193,15 +199,12 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage') }) {
     exclude group: "io.netty", module: "netty"
     exclude group: "log4j", module: "log4j"
     exclude group: "org.springframework.boot", module: "spring-boot-starter-logging"
-    exclude group: "ch.qos.logback", module: "logback-classic"
     exclude group: "org.apache.logging.log4j", module: "log4j-to-slf4j"
     exclude group: "com.vaadin.external.google", module: "android-json"
     exclude group: "org.slf4j", module: "slf4j-reload4j"
     exclude group: "org.slf4j", module: "slf4j-log4j12"
     exclude group: "org.slf4j", module: "slf4j-nop"
     exclude group: "org.slf4j", module: "slf4j-ext"
-    exclude group: "org.slf4j", module: "jul-to-slf4j"
-    exclude group: "org.slf4j", module: "jcl-over-toslf4j"
   }
 }
 

datahub-frontend/play.gradle

@@ -55,13 +55,14 @@ dependencies {
   testImplementation externalDependency.playTest
   testCompile externalDependency.testng
 
+  implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
   runtime externalDependency.guice
   runtime (externalDependency.playDocs) {
     exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12'
   }
   runtime externalDependency.playGuice
-  runtime externalDependency.logbackClassic
+  implementation externalDependency.logbackClassic
 
   annotationProcessor externalDependency.lombok
 }

datahub-graphql-core/build.gradle

@@ -15,6 +15,7 @@ dependencies {
     compile externalDependency.antlr4
     compile externalDependency.guava
 
+    implementation externalDependency.slf4jApi
     compileOnly externalDependency.lombok
     annotationProcessor externalDependency.lombok
 

datahub-ranger-plugin/build.gradle

@@ -28,7 +28,7 @@ dependencies {
         exclude group: "org.apache.htrace", module: "htrace-core4"
     }
     implementation externalDependency.hadoopCommon3
-    implementation externalDependency.log4jApi
+    implementation externalDependency.log4j12Api
 
     constraints {
         implementation(externalDependency.woodstoxCore) {

datahub-upgrade/build.gradle

@@ -14,7 +14,8 @@ dependencies {
     exclude group: 'com.nimbusds', module: 'nimbus-jose-jwt'
     exclude group: "org.apache.htrace", module: "htrace-core4"
   }
-  compile externalDependency.lombok
+  implementation externalDependency.slf4jApi
+  compileOnly externalDependency.lombok
   compile externalDependency.picocli
   compile externalDependency.parquet
   compile externalDependency.springBeans

entity-registry/build.gradle

@@ -4,7 +4,8 @@ dependencies {
   compile spec.product.pegasus.data
   compile spec.product.pegasus.generator
   compile project(path: ':metadata-models')
-  compile externalDependency.lombok
+  implementation externalDependency.slf4jApi
+  compileOnly externalDependency.lombok
   compile externalDependency.guava
   compile externalDependency.jacksonDataBind
   compile externalDependency.jacksonDataFormatYaml

ingestion-scheduler/build.gradle

@@ -4,12 +4,12 @@ dependencies {
     compile project(path: ':metadata-models')
     compile project(path: ':metadata-io')
     compile project(path: ':metadata-service:restli-client')
-    compile externalDependency.lombok
+    implementation externalDependency.slf4jApi
+    compileOnly externalDependency.lombok
     annotationProcessor externalDependency.lombok
 
     testCompile externalDependency.mockito
     testCompile externalDependency.testng
-    testAnnotationProcessor externalDependency.lombok
 
     constraints {
         implementation(externalDependency.log4jCore) {

li-utils/build.gradle

@@ -20,6 +20,7 @@ dependencies {
     }
     compile externalDependency.guava
 
+    implementation externalDependency.slf4jApi
     compileOnly externalDependency.lombok
     annotationProcessor externalDependency.lombok
 

metadata-dao-impl/kafka-producer/build.gradle

@@ -9,6 +9,7 @@ dependencies {
 
   compile externalDependency.kafkaClients
 
+  implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
 
   annotationProcessor externalDependency.lombok

metadata-ingestion/setup.py

@@ -221,7 +221,7 @@ data_lake_profiling = {
 
 delta_lake = {
     *s3_base,
-    "deltalake>=0.6.3",
+    "deltalake>=0.6.3, != 0.6.4",
 }
 
 powerbi_report_server = {"requests", "requests_ntlm"}

metadata-integration/java/datahub-client/build.gradle

@@ -42,7 +42,8 @@ dependencies {
     implementation(externalDependency.kafkaAvroSerializer) {
       exclude group: "org.apache.avro"
     }
-    
+
+    implementation externalDependency.slf4jApi
     compileOnly externalDependency.lombok
     annotationProcessor externalDependency.lombok
     testCompile externalDependency.mockito

metadata-integration/java/datahub-protobuf/build.gradle

@@ -35,6 +35,7 @@ dependencies {
     implementation externalDependency.gson
     implementation externalDependency.commonsCli
 
+    implementation externalDependency.slf4jApi
     compileOnly externalDependency.lombok
     annotationProcessor externalDependency.lombok
     testImplementation externalDependency.junitJupiterApi

metadata-integration/java/spark-lineage/build.gradle

@@ -47,6 +47,7 @@ dependencies {
     force = true
   } // required for org.apache.commons.io.Charsets that is used internally
 
+  implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
   annotationProcessor externalDependency.lombok
 
@@ -109,6 +110,7 @@ shadowJar {
   exclude('module-info.class', 'META-INF/versions/**')
 
   relocate 'com.fasterxml.jackson', 'datahub.shaded.jackson'
+  relocate 'org.slf4j','datahub.shaded.org.slf4j'
   relocate 'org.apache.http','datahub.spark2.shaded.http'
   relocate 'org.apache.commons.codec', 'datahub.spark2.shaded.o.a.c.codec'
   relocate 'org.apache.commons.compress', 'datahub.spark2.shaded.o.a.c.compress'

metadata-io/build.gradle

@@ -19,6 +19,8 @@ dependencies {
   compile spec.product.pegasus.generator
 
   compile externalDependency.dgraph4j exclude group: 'com.google.guava', module: 'guava'
+  implementation externalDependency.slf4jApi
+  runtime externalDependency.logbackClassic
   compileOnly externalDependency.lombok
   implementation externalDependency.commonsCollections
   compile externalDependency.datastaxOssNativeProtocol

metadata-jobs/mae-consumer/build.gradle

@@ -33,6 +33,7 @@ dependencies {
     compile externalDependency.springKafka
     compile externalDependency.springActuator
 
+    implementation externalDependency.slf4jApi
     compileOnly externalDependency.lombok
 
     annotationProcessor externalDependency.lombok

metadata-jobs/mce-consumer/build.gradle

@@ -30,6 +30,7 @@ dependencies {
     compile externalDependency.springKafka
     compile externalDependency.springActuator
 
+    implementation externalDependency.slf4jApi
     compileOnly externalDependency.lombok
 
     annotationProcessor externalDependency.lombok

metadata-jobs/pe-consumer/build.gradle

@@ -18,6 +18,7 @@ dependencies {
   }
   compile externalDependency.springKafka
   compile externalDependency.springActuator
+  implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
   annotationProcessor externalDependency.lombok
   runtime externalDependency.logbackClassic

metadata-models-validator/build.gradle

@@ -9,5 +9,7 @@ dependencies {
   compile externalDependency.findbugsAnnotations
   compile externalDependency.guava
 
+  implementation externalDependency.slf4jApi
+  runtimeOnly externalDependency.logbackClassic
   compileOnly externalDependency.lombok
 }

metadata-service/auth-api/build.gradle

@@ -3,6 +3,7 @@ apply plugin: 'java'
 dependencies {
   compile project(path: ':metadata-models')
   compile externalDependency.guava
-  compile externalDependency.lombok
+  implementation externalDependency.slf4jApi
+  compileOnly externalDependency.lombok
   annotationProcessor externalDependency.lombok
 }

metadata-service/auth-filter/build.gradle

@@ -5,7 +5,8 @@ dependencies {
   compile project(path: ':metadata-service:factories')
 
   compile externalDependency.servletApi
-  compile externalDependency.lombok
+  implementation externalDependency.slf4jApi
+  compileOnly externalDependency.lombok
   compile externalDependency.springWeb
 
   annotationProcessor externalDependency.lombok

metadata-service/auth-impl/build.gradle

@@ -14,7 +14,8 @@ dependencies {
   runtimeOnly 'io.jsonwebtoken:jjwt-impl:0.11.2',
       'io.jsonwebtoken:jjwt-jackson:0.11.2'
 
-  compile externalDependency.lombok
+  implementation externalDependency.slf4jApi
+  compileOnly externalDependency.lombok
 
   annotationProcessor externalDependency.lombok
 

metadata-service/auth-ranger-impl/build.gradle

@@ -28,7 +28,8 @@ dependencies {
     implementation 'org.apache.logging.log4j:log4j-1.2-api:2.17.1'
     implementation 'rome:rome:1.0'
     runtimeOnly externalDependency.jna
-    compile externalDependency.lombok
+    implementation externalDependency.slf4jApi
+    compileOnly externalDependency.lombok
 
     annotationProcessor externalDependency.lombok
     testCompile externalDependency.mockito

metadata-service/auth-servlet-impl/build.gradle

@@ -10,7 +10,8 @@ dependencies {
   compile externalDependency.graphqlJava
   compile externalDependency.springBeans
   compile externalDependency.springContext
-  compile externalDependency.lombok
+  implementation externalDependency.slf4jApi
+  compileOnly externalDependency.lombok
 
   annotationProcessor externalDependency.lombok
 }

metadata-service/factories/build.gradle

@@ -15,7 +15,7 @@ dependencies {
   compile externalDependency.gson
   compile externalDependency.kafkaClients
   compile externalDependency.kafkaAvroSerde
-  compile externalDependency.lombok
+  compileOnly externalDependency.lombok
   compile externalDependency.servletApi
   compile externalDependency.springBeans
   compile externalDependency.springBootAutoconfigure
@@ -31,6 +31,7 @@ dependencies {
   compile spec.product.pegasus.restliSpringBridge
 
   testImplementation externalDependency.springBootTest
+
   testCompile externalDependency.mockito
   testCompile externalDependency.testng
 

metadata-service/graphql-servlet-impl/build.gradle

@@ -11,7 +11,8 @@ dependencies {
   compile externalDependency.graphqlJava
   compile externalDependency.springBeans
   compile externalDependency.springContext
-  compile externalDependency.lombok
+  implementation externalDependency.slf4jApi
+  compileOnly externalDependency.lombok
 
   annotationProcessor externalDependency.lombok
 }

metadata-service/openapi-servlet/build.gradle

@@ -13,8 +13,10 @@ dependencies {
   compile externalDependency.springWebMVC
   compile externalDependency.springBeans
   compile externalDependency.springContext
-  compile externalDependency.lombok
+  implementation externalDependency.slf4jApi
-  compile externalDependency.antlr4
+  compileOnly externalDependency.lombok
+  implementation externalDependency.antlr4Runtime
+  implementation externalDependency.antlr4
 
   annotationProcessor externalDependency.lombok
 

metadata-service/restli-client/build.gradle

@@ -7,7 +7,8 @@ dependencies {
   compile project(':metadata-events:mxe-schemas')
   compile project(':metadata-utils')
 
-  compile externalDependency.lombok
+  implementation externalDependency.slf4jApi
+  compileOnly externalDependency.lombok
   annotationProcessor externalDependency.lombok
 
   compile spec.product.pegasus.restliClient

metadata-service/restli-servlet-impl/build.gradle

@@ -40,6 +40,8 @@ dependencies {
   compile project(':metadata-utils')
   compile project(':metadata-io')
   compile spec.product.pegasus.restliServer
+  implementation externalDependency.slf4jApi
+  // This is compile and not compileOnly because of restli
   compile externalDependency.lombok
   compile externalDependency.neo4jJavaDriver
   compile externalDependency.opentelemetryAnnotations
@@ -61,7 +63,6 @@ dependencies {
   modelValidation project(path: ':metadata-models-validator')
   dataModel project(path: ':metadata-models', configuration: 'dataTemplate')
   dataModel project(path: ':li-utils', configuration: 'dataTemplate')
-
 }
 
 task integrationTest(type: Test) {

metadata-service/war/build.gradle

@@ -13,9 +13,9 @@ dependencies {
   runtime project(':metadata-jobs:mae-consumer')
   runtime project(':metadata-jobs:pe-consumer')
 
+  runtime externalDependency.logbackClassic
   runtime externalDependency.awsSecretsManagerJdbc
   runtime externalDependency.h2
-  runtime externalDependency.logbackClassic
   runtime externalDependency.mariadbConnector
   runtime externalDependency.mysqlConnector
   runtime externalDependency.postgresql
@@ -28,15 +28,15 @@ dependencies {
 }
 
 configurations {
-  jetty8
+  jetty9
 }
 
 dependencies {
-  jetty8 "org.eclipse.jetty:jetty-runner:9.4.46.v20220331"
+  jetty9 "org.eclipse.jetty:jetty-runner:9.4.46.v20220331"
 }
 
 task run(type: JavaExec, dependsOn: build) {
   main = "org.eclipse.jetty.runner.Runner"
   args = [war.archivePath]
-  classpath configurations.jetty8
+  classpath configurations.jetty9
 }

metadata-utils/build.gradle

@@ -18,6 +18,7 @@ dependencies {
   compile project(':metadata-events:mxe-avro-1.7')
   compile project(':metadata-events:mxe-utils-avro-1.7')
 
+  implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
 
   annotationProcessor externalDependency.lombok