plugins { id 'org.springframework.boot' id 'java' } apply from: "../gradle/versioning/versioning.gradle" apply from: "../gradle/coverage/java-coverage.gradle" apply from: "../gradle/docker/docker.gradle" ext { docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry docker_repo = 'datahub-upgrade' } dependencies { implementation project(':metadata-io') implementation project(':metadata-service:factories') implementation project(':metadata-service:restli-client-api') implementation project(':metadata-service:configuration') implementation project(':metadata-dao-impl:kafka-producer') implementation externalDependency.charle implementation externalDependency.mustache implementation externalDependency.javaxInject implementation(externalDependency.hadoopClient) { exclude group: 'net.minidev', module: 'json-smart' exclude group: 'com.nimbusds', module: 'nimbus-jose-jwt' exclude group: "org.apache.htrace", module: "htrace-core4" exclude group: "org.eclipse.jetty" exclude group: "org.apache.hadoop.thirdparty", module: "hadoop-shaded-protobuf_3_7" exclude group: "com.charleskorn.kaml", module:"kaml" } constraints { implementation(externalDependency.hadoopCommon3) { because("previous versions are vulnerable to CVE-2021-37404") } implementation(externalDependency.snakeYaml) { because("previous versions are vulnerable to CVE-2022-25857") } implementation(externalDependency.woodstoxCore) { because("previous versions are vulnerable to CVE-2022-40151-2") } implementation(externalDependency.jettison) { because("previous versions are vulnerable") } implementation(externalDependency.guava) { because("CVE-2023-2976") } implementation('io.airlift:aircompressor:0.27') { because("CVE-2024-36114") } implementation('dnsjava:dnsjava:3.6.1') { because("CVE-2024-25638") } implementation('commons-beanutils:commons-beanutils:1.11.0') { because("CVE-2025-48734") } } // mock internal schema registry implementation externalDependency.kafkaAvroSerde implementation externalDependency.kafkaAvroSerializer implementation externalDependency.slf4jApi compileOnly externalDependency.lombok implementation externalDependency.picocli implementation externalDependency.parquet implementation externalDependency.protobuf implementation externalDependency.springBeans implementation externalDependency.springBootAutoconfigure implementation externalDependency.springCore implementation externalDependency.springKafka implementation externalDependency.kafkaClients runtimeOnly externalDependency.opentelemetryExporter runtimeOnly externalDependency.openTelemetryExporterLogging runtimeOnly externalDependency.openTelemetryExporterCommon runtimeOnly externalDependency.logbackClassic runtimeOnly externalDependency.mariadbConnector runtimeOnly externalDependency.mysqlConnector runtimeOnly externalDependency.postgresql implementation externalDependency.awsMskIamAuth implementation externalDependency.azureIdentityExtensions implementation externalDependency.azureIdentity implementation platform(externalDependency.jacksonBom) implementation externalDependency.jacksonJsr310 annotationProcessor externalDependency.lombok annotationProcessor externalDependency.picocli testImplementation externalDependency.springBootTest testImplementation externalDependency.mockito testImplementation externalDependency.testng testImplementation 'uk.org.webcompere:system-stubs-testng:2.1.7' testRuntimeOnly externalDependency.logbackClassic testImplementation externalDependency.h2 testImplementation testFixtures(project(':metadata-io')) constraints { implementation(implementation externalDependency.parquetHadoop) { because("CVE-2022-42003") } } } bootJar { mainClass = 'com.linkedin.datahub.upgrade.UpgradeCliApplication' archiveFileName = "${project.name}.jar" } bootRun { environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml" environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true" environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true" environment "SERVER_PORT", "8083" args += ["-u", "SystemUpdate"] } /** * Runs SystemUpdate on locally running system */ task run(type: Exec) { dependsOn bootJar group = "Execution" description = "Run the datahub-upgrade SystemUpdate process locally." environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml" environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true" environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true" commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n", "-jar", "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdate" } task runCron(type: Exec) { dependsOn bootJar group = "Execution" description = "Run the datahub-upgrade SystemUpdate CRON process locally." environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml" environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true" environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true" commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n", "-jar", "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdateCron" } task runCronDryRun(type: Exec) { dependsOn bootJar group = "Execution" description = "Run the datahub-upgrade SystemUpdate CRON process locally." environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml" environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true" environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true" commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n", "-jar", "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdateCron", "-a", "dryRun=true" } /** * Runs RestoreIndices on locally running system. The batchSize are set to * test the process with pagination and not designed for optimal performance. */ task runRestoreIndices(type: Exec) { dependsOn bootJar group = "Execution" description = "Run the restore indices process locally." environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml" commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n", "-jar", "-Dkafka.schemaRegistry.url=http://localhost:8080/schema-registry/api", "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100", "-a", "createDefaultAspects=true" } task runRestoreIndicesUrn(type: Exec) { dependsOn bootJar group = "Execution" description = "Run the restore indices process locally." environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml" commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n", "-jar", "-Dkafka.schemaRegistry.url=http://localhost:8080/schema-registry/api", "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100", "-a", "urnBasedPagination=true" } docker { dependsOn(bootJar) name "${docker_registry}/${docker_repo}:${versionTag}" dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile") files bootJar.outputs.files files fileTree(rootProject.projectDir) { include '.dockerignore' include 'docker/monitoring/*' include "docker/${docker_repo}/*" include 'metadata-models/src/main/resources/*' }.exclude { i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden()) } additionalTag("Debug", "${docker_registry}/${docker_repo}:debug") // Add build args if they are defined (needed for some CI or enterprise environments) def dockerBuildArgs = [:] if (project.hasProperty('alpineApkRepositoryUrl')) { dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl') } if (project.hasProperty('githubMirrorUrl')) { dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl') } if (project.hasProperty('mavenCentralRepositoryUrl')) { dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl') } if (dockerBuildArgs.size() > 0) { buildArgs(dockerBuildArgs) } }