datahub/metadata-ingestion/build.gradle

plugins {
  id 'base'
}

ext {
  python_executable = 'python3'
  venv_name = 'venv'
}

if (!project.hasProperty("extra_pip_requirements")) {
    ext.extra_pip_requirements = ""
}

def get_coverage_arg(test_name) {
  return "--cov-report term --cov-report xml:coverage_${test_name}.xml "
}

task checkPythonVersion(type: Exec) {
  commandLine python_executable, '-c',
    'import sys; assert (3, 11) > sys.version_info >= (3, 7), f"Python version {sys.version_info[:2]} not allowed"'
}

task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
  inputs.file file('setup.py')
  outputs.dir("${venv_name}")
  commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && ${venv_name}/bin/python -m pip install --upgrade pip wheel 'setuptools>=63.0.0'"
}

task runPreFlightScript(type: Exec, dependsOn: environmentSetup) {
  def sentinel_file = ".preflight_sentinel"
  outputs.file(sentinel_file)
  commandLine "scripts/datahub_preflight.sh"
  commandLine 'bash', '-c', "touch ${sentinel_file}"
}

task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) {
  def sentinel_file = "${venv_name}/.build_install_package_only_sentinel"
  inputs.file file('setup.py')
  outputs.dir("${venv_name}")
  outputs.file(sentinel_file)
  commandLine 'bash', '-x', '-c', "${venv_name}/bin/pip install -e ."
  commandLine 'bash', '-c', "touch ${sentinel_file}"
}

task installPackage(type: Exec, dependsOn: installPackageOnly) {
  inputs.file file('setup.py')
  outputs.dir("${venv_name}")
  commandLine 'bash', '-x', '-c', "${venv_name}/bin/pip install -e . ${extra_pip_requirements}"
}

task codegen(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadata-events:mxe-schemas:build']) {
  inputs.files(project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc"))
  outputs.dir('src/datahub/metadata')
  commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/codegen.sh"
}

task install(dependsOn: [installPackage, codegen])

task installDev(type: Exec, dependsOn: [install]) {
  def sentinel_file = "${venv_name}/.build_install_dev_sentinel"
  inputs.file file('setup.py')
  outputs.dir("${venv_name}")
  outputs.file(sentinel_file)
  commandLine 'bash', '-c',
    "source ${venv_name}/bin/activate && set -x && " +
    "${venv_name}/bin/pip install -e .[dev] ${extra_pip_requirements} && " +
    "./scripts/install-sqlalchemy-stubs.sh && " +
    "touch ${sentinel_file}"
}

task installAll(type: Exec, dependsOn: [install]) {
  def sentinel_file = "${venv_name}/.build_install_all_sentinel"
  inputs.file file('setup.py')
  outputs.dir("${venv_name}")
  outputs.file(sentinel_file)
  commandLine 'bash', '-c',
    "source ${venv_name}/bin/activate && set -x && " +
    "${venv_name}/bin/pip install -e .[all] ${extra_pip_requirements} && " +
    "./scripts/install-sqlalchemy-stubs.sh && " +
    "touch ${sentinel_file}"
}


task modelDocGen(type: Exec, dependsOn: [codegen]) {
  inputs.files(
    file('scripts/modeldocgen.py'),
    project.fileTree(dir: "../metadata-models/docs/entities/", include: "**/*.md"),
    project.fileTree(dir: "examples/", include: "**/*.py"),
    project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc")
  )
  outputs.dir('../docs/generated/metamodel')
  commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/modeldocgen.sh"
}

task modelDocUpload(type: Exec, dependsOn: [modelDocGen]) {
    commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/modeldocupload.sh"
}


task lint(type: Exec, dependsOn: installDev) {
  /*
    The find/sed combo below is a temporary work-around for the following mypy issue with airflow 2.2.0:
   "venv/lib/python3.8/site-packages/airflow/_vendor/connexion/spec.py:169: error: invalid syntax".
   */
  commandLine 'bash', '-c',
    "find ${venv_name}/lib -path *airflow/_vendor/connexion/spec.py -exec sed -i.bak -e '169,169s/  # type: List\\[str\\]//g' {} \\; && " +
    "source ${venv_name}/bin/activate && set -x && " +
    "black --check --diff src/ tests/ examples/ && " +
    "isort --check --diff src/ tests/ examples/ && " +
    "flake8 --count --statistics src/ tests/ examples/ && " +
    "mypy --show-traceback --show-error-codes src/ tests/ examples/"
}
task lintFix(type: Exec, dependsOn: installDev) {
  commandLine 'bash', '-c',
    "source ${venv_name}/bin/activate && set -x && " +
    "./scripts/install-sqlalchemy-stubs.sh && " +
    "black src/ tests/ examples/ && " +
    "isort src/ tests/ examples/ && " +
    "flake8 src/ tests/ examples/ && " +
    "mypy --show-traceback --show-error-codes src/ tests/ examples/"
}

task testQuick(type: Exec, dependsOn: [installDev, ':metadata-models:generateJsonSchema']) {
  // We can't enforce the coverage requirements if we run a subset of the tests.
  inputs.files(project.fileTree(dir: "src/", include: "**/*.py"))
  inputs.files(project.fileTree(dir: "tests/"))
  outputs.dir("${venv_name}")
  def cvg_arg = get_coverage_arg("quick")
  commandLine 'bash', '-c',
    "source ${venv_name}/bin/activate && pytest ${cvg_arg} --durations=20 -m 'not integration and not integration_batch_1 and not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
}

task installDevTest(type: Exec, dependsOn: [install]) {
  def sentinel_file = "${venv_name}/.build_install_dev_test_sentinel"
  inputs.file file('setup.py')
  outputs.dir("${venv_name}")
  outputs.file(sentinel_file)
  commandLine 'bash', '-c',
    "${venv_name}/bin/pip install -e .[dev,integration-tests] ${extra_pip_requirements} && touch ${sentinel_file}"
}

def testFile = hasProperty('testFile') ? testFile : 'unknown'
task testSingle(dependsOn: [installDevTest]) {
  doLast {
  if (testFile != 'unknown')  {
    exec {
      commandLine 'bash', '-c',
        "source ${venv_name}/bin/activate && pytest ${testFile}"
    }
  } else {
   throw new GradleException("No file provided. Use -PtestFile=<test_file>")
    }
  }
}

task testIntegration(type: Exec, dependsOn: [installDevTest]) {
  def cvg_arg = get_coverage_arg("int")
  commandLine 'bash', '-c',
    "source ${venv_name}/bin/activate && pytest ${cvg_arg} --durations=50 -m 'integration' -vv --continue-on-collection-errors --junit-xml=junit.integration.xml"
}

task testIntegrationBatch1(type: Exec, dependsOn: [installDevTest]) {
  def cvg_arg = get_coverage_arg("intBatch1")
  commandLine 'bash', '-c',
    "source ${venv_name}/bin/activate && pytest ${cvg_arg} --durations=50 -m 'integration_batch_1' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch1.xml"
}

task testFull(type: Exec, dependsOn: [installDevTest]) {
  commandLine 'bash', '-c',
    "source ${venv_name}/bin/activate && pytest --durations=50 -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
}

task testSlowIntegration(type: Exec, dependsOn: [installDevTest]) {
  def cvg_arg = get_coverage_arg("intSlow")
  commandLine 'bash', '-c',
    "source ${venv_name}/bin/activate && pytest ${cvg_arg} --durations=20 -m 'slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.slow.integration.xml"
}

task docGen(type: Exec, dependsOn: [codegen, installDevTest]) {
  commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/docgen.sh"
}
task buildWheel(type: Exec, dependsOn: [install, codegen]) {
  commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
}



task cleanPythonCache(type: Exec) {
  commandLine 'bash', '-c',
    "find src tests -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete"
}

build.dependsOn install
check.dependsOn lint
check.dependsOn testQuick

clean {
  delete venv_name
  delete 'build'
  delete 'dist'
  delete 'src/datahub/metadata'
  delete '../docs/generated'
  delete 'generated'
  delete '.mypy_cache'
  delete '.pytest_cache'
  delete '.preflight_sentinel'
}
clean.dependsOn cleanPythonCache

idea {
  module {
    sourceDirs += file('src')
    testSourceDirs += file('tests')
  }
}