datahub/datahub-upgrade/build.gradle

plugins {
  id 'org.springframework.boot'
  id 'java'
}

apply from: "../gradle/versioning/versioning.gradle"
apply from: "../gradle/coverage/java-coverage.gradle"
apply from: "../gradle/docker/docker.gradle"

ext {
  docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry
  docker_repo = 'datahub-upgrade'
}

dependencies {
  implementation project(':metadata-io')
  implementation project(':metadata-service:factories')
  implementation project(':metadata-service:restli-client-api')
  implementation project(':metadata-service:configuration')
  implementation project(':metadata-dao-impl:kafka-producer')
  implementation externalDependency.charle

  implementation externalDependency.mustache
  implementation externalDependency.javaxInject
  implementation(externalDependency.hadoopClient) {
    exclude group: 'net.minidev', module: 'json-smart'
    exclude group: 'com.nimbusds', module: 'nimbus-jose-jwt'
    exclude group: "org.apache.htrace", module: "htrace-core4"
    exclude group: "org.eclipse.jetty"
    exclude group: "org.apache.hadoop.thirdparty", module: "hadoop-shaded-protobuf_3_7"
    exclude group: "com.charleskorn.kaml", module:"kaml"

  }

  constraints {
    implementation(externalDependency.hadoopCommon3) {
      because("previous versions are vulnerable to CVE-2021-37404")
    }
    implementation(externalDependency.snakeYaml) {
      because("previous versions are vulnerable to CVE-2022-25857")
    }
    implementation(externalDependency.woodstoxCore) {
      because("previous versions are vulnerable to CVE-2022-40151-2")
    }
    implementation(externalDependency.jettison) {
      because("previous versions are vulnerable")
    }
    implementation(externalDependency.guava) {
      because("CVE-2023-2976")
    }
    implementation('io.airlift:aircompressor:0.27') {
      because("CVE-2024-36114")
    }
    implementation('dnsjava:dnsjava:3.6.1') {
      because("CVE-2024-25638")
    }
    implementation('commons-beanutils:commons-beanutils:1.11.0') {
      because("CVE-2025-48734")
    }
  }


  // mock internal schema registry
  implementation externalDependency.kafkaAvroSerde
  implementation externalDependency.kafkaAvroSerializer

  implementation externalDependency.slf4jApi
  compileOnly externalDependency.lombok
  implementation externalDependency.picocli
  implementation externalDependency.parquet
  implementation externalDependency.protobuf
  implementation externalDependency.springBeans
  implementation externalDependency.springBootAutoconfigure
  implementation externalDependency.springCore
  implementation externalDependency.springKafka
  implementation externalDependency.kafkaClients
  runtimeOnly externalDependency.opentelemetryExporter
  runtimeOnly externalDependency.openTelemetryExporterLogging
  runtimeOnly externalDependency.openTelemetryExporterCommon

  runtimeOnly externalDependency.logbackClassic
  runtimeOnly externalDependency.mariadbConnector
  runtimeOnly externalDependency.mysqlConnector
  runtimeOnly externalDependency.postgresql

  implementation externalDependency.awsMskIamAuth
  implementation externalDependency.azureIdentityExtensions
  implementation externalDependency.azureIdentity

  implementation platform(externalDependency.jacksonBom)
  implementation externalDependency.jacksonJsr310

  annotationProcessor externalDependency.lombok
  annotationProcessor externalDependency.picocli

  testImplementation externalDependency.springBootTest
  testImplementation externalDependency.mockito
  testImplementation externalDependency.testng
  testImplementation 'uk.org.webcompere:system-stubs-testng:2.1.7'
  testRuntimeOnly externalDependency.logbackClassic

  testImplementation externalDependency.h2
  testImplementation testFixtures(project(':metadata-io'))

  constraints {
    implementation(externalDependency.parquetHadoop) {
      because("CVE-2022-42003")
    }
  }
}

bootJar {
  mainClass = 'com.linkedin.datahub.upgrade.UpgradeCliApplication'
  archiveFileName = "${project.name}.jar"
}

bootRun {
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
  environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
  environment "SERVER_PORT", "8083"
  args += ["-u", "SystemUpdate"]
}

/**
 * Runs SystemUpdate on locally running system
 */
task run(type: Exec) {
  dependsOn bootJar
  group = "Execution"
  description = "Run the datahub-upgrade SystemUpdate process locally."
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
  environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
  environment "SCHEMA_REGISTRY_TYPE", "INTERNAL"
  commandLine "java",
          "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
          "-jar",
          "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdate"
}

/**
 * Runs the non-blocking system updates locally (includes lineage index fields backfill)
 * Sets up environment variables for local execution
 */
task runNonBlocking(type: Exec) {
  dependsOn bootJar
  group = "Execution"
  description = "Run the non-blocking system updates locally (includes lineage index fields backfill)."
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
  environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
  environment "BOOTSTRAP_SYSTEM_UPDATE_EXECUTOR_POOLS_ENABLED", "false"
  environment "SCHEMA_REGISTRY_TYPE", "INTERNAL"
  commandLine "java",
          "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
          "-jar",
          "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdateNonBlocking"
}

task runCron(type: Exec) {
  dependsOn bootJar
  group = "Execution"
  description = "Run the datahub-upgrade SystemUpdate CRON process locally."
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
  environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
  commandLine "java",
          "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
          "-jar",
          "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdateCron"
}

task runCronDryRun(type: Exec) {
  dependsOn bootJar
  group = "Execution"
  description = "Run the datahub-upgrade SystemUpdate CRON process locally."
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
  environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
  commandLine "java",
          "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
          "-jar",
          "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdateCron", "-a", "dryRun=true"
}

/**
 * Runs RestoreIndices on locally running system.
 * This is useful for debugging or special situations to reindex, the index to reindex is given in -u index= argument"
 */
task runReindexDebug(type: Exec) {
  dependsOn bootJar
  group = "Execution"
  description = "Run the datahub-upgrade SystemUpdate ReindexDebug"
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
  environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
  commandLine "java",
          "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
          "-jar",
          "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "ReindexDebug", "-a", "index=datahubpolicyindex_v2"
}

/**
 * Runs RestoreIndices on locally running system. The batchSize are set to
 * test the process with pagination and not designed for optimal performance.
 */
task runRestoreIndices(type: Exec) {
  dependsOn bootJar
  group = "Execution"
  description = "Run the restore indices process locally."
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
          "-jar",
          "-Dkafka.schemaRegistry.url=http://localhost:8080/schema-registry/api",
          "-Dserver.port=8083",
          bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100", "-a", "createDefaultAspects=true"
}

task runRestoreIndicesUrn(type: Exec) {
  dependsOn bootJar
  group = "Execution"
  description = "Run the restore indices process locally."
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
          "-jar",
          "-Dkafka.schemaRegistry.url=http://localhost:8080/schema-registry/api",
          "-Dserver.port=8083",
          bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100", "-a", "urnBasedPagination=true"
}

docker {
  dependsOn(bootJar)
  name "${docker_registry}/${docker_repo}:${versionTag}"
  dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
  files bootJar.outputs.files
  files fileTree(rootProject.projectDir) {
    include '.dockerignore'
    include 'docker/monitoring/*'
    include "docker/${docker_repo}/*"
    include 'metadata-models/src/main/resources/*'
  }.exclude {
    i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
  }
  additionalTag("Debug", "${docker_registry}/${docker_repo}:debug")

  // Add build args if they are defined (needed for some CI or enterprise environments)
  def dockerBuildArgs = [:]
  if (project.hasProperty('alpineApkRepositoryUrl')) {
    dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
  }
  if (project.hasProperty('githubMirrorUrl')) {
    dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
  }
  if (project.hasProperty('mavenCentralRepositoryUrl')) {
    dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
  }

  if (dockerBuildArgs.size() > 0) {
    buildArgs(dockerBuildArgs)
  }
}