mirror of
https://github.com/datahub-project/datahub.git
synced 2025-10-30 18:26:58 +00:00
263 lines
10 KiB
Groovy
263 lines
10 KiB
Groovy
plugins {
|
|
id 'org.springframework.boot'
|
|
id 'java'
|
|
}
|
|
|
|
apply from: "../gradle/versioning/versioning.gradle"
|
|
apply from: "../gradle/coverage/java-coverage.gradle"
|
|
apply from: "../gradle/docker/docker.gradle"
|
|
|
|
ext {
|
|
docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry
|
|
docker_repo = 'datahub-upgrade'
|
|
}
|
|
|
|
dependencies {
|
|
implementation project(':metadata-io')
|
|
implementation project(':metadata-service:factories')
|
|
implementation project(':metadata-service:restli-client-api')
|
|
implementation project(':metadata-service:configuration')
|
|
implementation project(':metadata-dao-impl:kafka-producer')
|
|
implementation externalDependency.charle
|
|
|
|
implementation externalDependency.mustache
|
|
implementation externalDependency.javaxInject
|
|
implementation(externalDependency.hadoopClient) {
|
|
exclude group: 'net.minidev', module: 'json-smart'
|
|
exclude group: 'com.nimbusds', module: 'nimbus-jose-jwt'
|
|
exclude group: "org.apache.htrace", module: "htrace-core4"
|
|
exclude group: "org.eclipse.jetty"
|
|
exclude group: "org.apache.hadoop.thirdparty", module: "hadoop-shaded-protobuf_3_7"
|
|
exclude group: "com.charleskorn.kaml", module:"kaml"
|
|
|
|
}
|
|
|
|
constraints {
|
|
implementation(externalDependency.hadoopCommon3) {
|
|
because("previous versions are vulnerable to CVE-2021-37404")
|
|
}
|
|
implementation(externalDependency.snakeYaml) {
|
|
because("previous versions are vulnerable to CVE-2022-25857")
|
|
}
|
|
implementation(externalDependency.woodstoxCore) {
|
|
because("previous versions are vulnerable to CVE-2022-40151-2")
|
|
}
|
|
implementation(externalDependency.jettison) {
|
|
because("previous versions are vulnerable")
|
|
}
|
|
implementation(externalDependency.guava) {
|
|
because("CVE-2023-2976")
|
|
}
|
|
implementation('io.airlift:aircompressor:0.27') {
|
|
because("CVE-2024-36114")
|
|
}
|
|
implementation('dnsjava:dnsjava:3.6.1') {
|
|
because("CVE-2024-25638")
|
|
}
|
|
implementation('commons-beanutils:commons-beanutils:1.11.0') {
|
|
because("CVE-2025-48734")
|
|
}
|
|
}
|
|
|
|
|
|
// mock internal schema registry
|
|
implementation externalDependency.kafkaAvroSerde
|
|
implementation externalDependency.kafkaAvroSerializer
|
|
|
|
implementation externalDependency.slf4jApi
|
|
compileOnly externalDependency.lombok
|
|
implementation externalDependency.picocli
|
|
implementation externalDependency.parquet
|
|
implementation externalDependency.protobuf
|
|
implementation externalDependency.springBeans
|
|
implementation externalDependency.springBootAutoconfigure
|
|
implementation externalDependency.springCore
|
|
implementation externalDependency.springKafka
|
|
implementation externalDependency.kafkaClients
|
|
runtimeOnly externalDependency.opentelemetryExporter
|
|
runtimeOnly externalDependency.openTelemetryExporterLogging
|
|
runtimeOnly externalDependency.openTelemetryExporterCommon
|
|
|
|
runtimeOnly externalDependency.logbackClassic
|
|
runtimeOnly externalDependency.mariadbConnector
|
|
runtimeOnly externalDependency.mysqlConnector
|
|
runtimeOnly externalDependency.postgresql
|
|
|
|
implementation externalDependency.awsMskIamAuth
|
|
implementation externalDependency.azureIdentityExtensions
|
|
implementation externalDependency.azureIdentity
|
|
|
|
implementation platform(externalDependency.jacksonBom)
|
|
implementation externalDependency.jacksonJsr310
|
|
|
|
annotationProcessor externalDependency.lombok
|
|
annotationProcessor externalDependency.picocli
|
|
|
|
testImplementation externalDependency.springBootTest
|
|
testImplementation externalDependency.mockito
|
|
testImplementation externalDependency.testng
|
|
testImplementation 'uk.org.webcompere:system-stubs-testng:2.1.7'
|
|
testRuntimeOnly externalDependency.logbackClassic
|
|
|
|
testImplementation externalDependency.h2
|
|
testImplementation testFixtures(project(':metadata-io'))
|
|
|
|
constraints {
|
|
implementation(externalDependency.parquetHadoop) {
|
|
because("CVE-2022-42003")
|
|
}
|
|
}
|
|
}
|
|
|
|
bootJar {
|
|
mainClass = 'com.linkedin.datahub.upgrade.UpgradeCliApplication'
|
|
archiveFileName = "${project.name}.jar"
|
|
}
|
|
|
|
bootRun {
|
|
environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
|
|
environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
|
|
environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
|
|
environment "SERVER_PORT", "8083"
|
|
args += ["-u", "SystemUpdate"]
|
|
}
|
|
|
|
/**
|
|
* Runs SystemUpdate on locally running system
|
|
*/
|
|
task run(type: Exec) {
|
|
dependsOn bootJar
|
|
group = "Execution"
|
|
description = "Run the datahub-upgrade SystemUpdate process locally."
|
|
environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
|
|
environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
|
|
environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
|
|
environment "SCHEMA_REGISTRY_TYPE", "INTERNAL"
|
|
commandLine "java",
|
|
"-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
|
|
"-jar",
|
|
"-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdate"
|
|
}
|
|
|
|
/**
|
|
* Runs the non-blocking system updates locally (includes lineage index fields backfill)
|
|
* Sets up environment variables for local execution
|
|
*/
|
|
task runNonBlocking(type: Exec) {
|
|
dependsOn bootJar
|
|
group = "Execution"
|
|
description = "Run the non-blocking system updates locally (includes lineage index fields backfill)."
|
|
environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
|
|
environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
|
|
environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
|
|
environment "BOOTSTRAP_SYSTEM_UPDATE_EXECUTOR_POOLS_ENABLED", "false"
|
|
environment "SCHEMA_REGISTRY_TYPE", "INTERNAL"
|
|
commandLine "java",
|
|
"-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
|
|
"-jar",
|
|
"-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdateNonBlocking"
|
|
}
|
|
|
|
task runCron(type: Exec) {
|
|
dependsOn bootJar
|
|
group = "Execution"
|
|
description = "Run the datahub-upgrade SystemUpdate CRON process locally."
|
|
environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
|
|
environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
|
|
environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
|
|
commandLine "java",
|
|
"-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
|
|
"-jar",
|
|
"-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdateCron"
|
|
}
|
|
|
|
task runCronDryRun(type: Exec) {
|
|
dependsOn bootJar
|
|
group = "Execution"
|
|
description = "Run the datahub-upgrade SystemUpdate CRON process locally."
|
|
environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
|
|
environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
|
|
environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
|
|
commandLine "java",
|
|
"-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
|
|
"-jar",
|
|
"-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdateCron", "-a", "dryRun=true"
|
|
}
|
|
|
|
/**
|
|
* Runs RestoreIndices on locally running system.
|
|
* This is useful for debugging or special situations to reindex, the index to reindex is given in -u index= argument"
|
|
*/
|
|
task runReindexDebug(type: Exec) {
|
|
dependsOn bootJar
|
|
group = "Execution"
|
|
description = "Run the datahub-upgrade SystemUpdate ReindexDebug"
|
|
environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
|
|
environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
|
|
environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
|
|
commandLine "java",
|
|
"-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
|
|
"-jar",
|
|
"-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "ReindexDebug", "-a", "index=datahubpolicyindex_v2"
|
|
}
|
|
|
|
/**
|
|
* Runs RestoreIndices on locally running system. The batchSize are set to
|
|
* test the process with pagination and not designed for optimal performance.
|
|
*/
|
|
task runRestoreIndices(type: Exec) {
|
|
dependsOn bootJar
|
|
group = "Execution"
|
|
description = "Run the restore indices process locally."
|
|
environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
|
|
commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
|
|
"-jar",
|
|
"-Dkafka.schemaRegistry.url=http://localhost:8080/schema-registry/api",
|
|
"-Dserver.port=8083",
|
|
bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100", "-a", "createDefaultAspects=true"
|
|
}
|
|
|
|
task runRestoreIndicesUrn(type: Exec) {
|
|
dependsOn bootJar
|
|
group = "Execution"
|
|
description = "Run the restore indices process locally."
|
|
environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
|
|
commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
|
|
"-jar",
|
|
"-Dkafka.schemaRegistry.url=http://localhost:8080/schema-registry/api",
|
|
"-Dserver.port=8083",
|
|
bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100", "-a", "urnBasedPagination=true"
|
|
}
|
|
|
|
docker {
|
|
dependsOn(bootJar)
|
|
name "${docker_registry}/${docker_repo}:${versionTag}"
|
|
dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
|
|
files bootJar.outputs.files
|
|
files fileTree(rootProject.projectDir) {
|
|
include '.dockerignore'
|
|
include 'docker/monitoring/*'
|
|
include "docker/${docker_repo}/*"
|
|
include 'metadata-models/src/main/resources/*'
|
|
}.exclude {
|
|
i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
|
|
}
|
|
additionalTag("Debug", "${docker_registry}/${docker_repo}:debug")
|
|
|
|
// Add build args if they are defined (needed for some CI or enterprise environments)
|
|
def dockerBuildArgs = [:]
|
|
if (project.hasProperty('alpineApkRepositoryUrl')) {
|
|
dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
|
|
}
|
|
if (project.hasProperty('githubMirrorUrl')) {
|
|
dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
|
|
}
|
|
if (project.hasProperty('mavenCentralRepositoryUrl')) {
|
|
dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
|
|
}
|
|
|
|
if (dockerBuildArgs.size() > 0) {
|
|
buildArgs(dockerBuildArgs)
|
|
}
|
|
}
|