mirror of
https://github.com/datahub-project/datahub.git
synced 2025-11-02 03:39:03 +00:00
243 lines
8.8 KiB
Groovy
243 lines
8.8 KiB
Groovy
plugins {
|
|
id 'base'
|
|
}
|
|
|
|
ext {
|
|
python_executable = 'python3'
|
|
venv_name = 'venv'
|
|
}
|
|
|
|
if (!project.hasProperty("extra_pip_requirements")) {
|
|
ext.extra_pip_requirements = ""
|
|
}
|
|
|
|
def get_coverage_arg(test_name) {
|
|
return "--cov-report xml:coverage_${test_name}.xml "
|
|
}
|
|
|
|
task checkPythonVersion(type: Exec) {
|
|
commandLine python_executable, '-c',
|
|
'import sys; assert (3, 11) > sys.version_info >= (3, 8), f"Python version {sys.version_info[:2]} not allowed"'
|
|
}
|
|
|
|
task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
|
|
def sentinel_file = "${venv_name}/.venv_environment_sentinel"
|
|
inputs.file file('setup.py')
|
|
outputs.file(sentinel_file)
|
|
commandLine 'bash', '-c',
|
|
"${python_executable} -m venv ${venv_name} && " +
|
|
"${venv_name}/bin/python -m pip install --upgrade pip uv wheel 'setuptools>=63.0.0' && " +
|
|
"touch ${sentinel_file}"
|
|
}
|
|
|
|
task runPreFlightScript(type: Exec, dependsOn: environmentSetup) {
|
|
def sentinel_file = ".preflight_sentinel"
|
|
outputs.file(sentinel_file)
|
|
commandLine 'bash', '-c',
|
|
"scripts/datahub_preflight.sh && " +
|
|
"touch ${sentinel_file}"
|
|
}
|
|
|
|
task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) {
|
|
def sentinel_file = "${venv_name}/.build_install_package_only_sentinel"
|
|
inputs.file file('setup.py')
|
|
outputs.file(sentinel_file)
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"uv pip install -e . &&" +
|
|
"touch ${sentinel_file}"
|
|
}
|
|
|
|
task installPackage(type: Exec, dependsOn: installPackageOnly) {
|
|
def sentinel_file = "${venv_name}/.build_install_package_sentinel"
|
|
inputs.file file('setup.py')
|
|
outputs.file(sentinel_file)
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"uv pip install -e . ${extra_pip_requirements} && " +
|
|
"touch ${sentinel_file}"
|
|
}
|
|
|
|
task codegen(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadata-events:mxe-schemas:build']) {
|
|
inputs.files(
|
|
project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc"),
|
|
project.fileTree(dir: "scripts"),
|
|
)
|
|
outputs.dir('src/datahub/metadata')
|
|
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/codegen.sh"
|
|
}
|
|
|
|
task customPackageGenerate(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadata-events:mxe-schemas:build']) {
|
|
def package_name = project.findProperty('package_name')
|
|
def package_version = project.findProperty('package_version')
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"uv pip install build && " +
|
|
"./scripts/custom_package_codegen.sh '${package_name}' '${package_version}'"
|
|
}
|
|
|
|
task install(dependsOn: [installPackage, codegen])
|
|
|
|
task installDev(type: Exec, dependsOn: [install]) {
|
|
def sentinel_file = "${venv_name}/.build_install_dev_sentinel"
|
|
inputs.file file('setup.py')
|
|
outputs.file(sentinel_file)
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"uv pip install -e .[dev] ${extra_pip_requirements} && " +
|
|
"touch ${sentinel_file}"
|
|
}
|
|
|
|
task installAll(type: Exec, dependsOn: [install]) {
|
|
def sentinel_file = "${venv_name}/.build_install_all_sentinel"
|
|
inputs.file file('setup.py')
|
|
outputs.file(sentinel_file)
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"uv pip install -e .[all] ${extra_pip_requirements} && " +
|
|
"touch ${sentinel_file}"
|
|
}
|
|
|
|
|
|
task modelDocGen(type: Exec, dependsOn: [codegen]) {
|
|
inputs.files(
|
|
file('scripts/modeldocgen.py'),
|
|
project.fileTree(dir: "../metadata-models/docs/entities/", include: "**/*.md"),
|
|
project.fileTree(dir: "examples/", include: "**/*.py"),
|
|
project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc")
|
|
)
|
|
outputs.dir('../docs/generated/metamodel')
|
|
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/modeldocgen.sh"
|
|
}
|
|
|
|
task modelDocUpload(type: Exec, dependsOn: [modelDocGen]) {
|
|
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/modeldocupload.sh"
|
|
}
|
|
|
|
|
|
task lint(type: Exec, dependsOn: installDev) {
|
|
/*
|
|
The find/sed combo below is a temporary work-around for the following mypy issue with airflow 2.2.0:
|
|
"venv/lib/python3.8/site-packages/airflow/_vendor/connexion/spec.py:169: error: invalid syntax".
|
|
*/
|
|
commandLine 'bash', '-c',
|
|
"find ${venv_name}/lib -path *airflow/_vendor/connexion/spec.py -exec sed -i.bak -e '169,169s/ # type: List\\[str\\]//g' {} \\; && " +
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"black --check --diff src/ tests/ examples/ && " +
|
|
"isort --check --diff src/ tests/ examples/ && " +
|
|
"flake8 --count --statistics src/ tests/ examples/ && " +
|
|
"mypy --show-traceback --show-error-codes src/ tests/ examples/"
|
|
}
|
|
task lintFix(type: Exec, dependsOn: installDev) {
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"black src/ tests/ examples/ && " +
|
|
"isort src/ tests/ examples/ && " +
|
|
"flake8 src/ tests/ examples/ && " +
|
|
"mypy --show-traceback --show-error-codes src/ tests/ examples/"
|
|
}
|
|
|
|
task testQuick(type: Exec, dependsOn: [installDev, ':metadata-models:generateJsonSchema']) {
|
|
// We can't enforce the coverage requirements if we run a subset of the tests.
|
|
inputs.files(project.fileTree(dir: "src/", include: "**/*.py"))
|
|
inputs.files(project.fileTree(dir: "tests/"))
|
|
outputs.dir("${venv_name}")
|
|
def cvg_arg = get_coverage_arg("quick")
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"pytest ${cvg_arg} tests/unit --random-order --durations=20 -m 'not integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
|
|
}
|
|
|
|
task installDevTest(type: Exec, dependsOn: [install]) {
|
|
def sentinel_file = "${venv_name}/.build_install_dev_test_sentinel"
|
|
inputs.file file('setup.py')
|
|
outputs.dir("${venv_name}")
|
|
outputs.file(sentinel_file)
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"uv pip install -e .[dev,integration-tests] ${extra_pip_requirements} && " +
|
|
"touch ${sentinel_file}"
|
|
}
|
|
|
|
def testFile = hasProperty('testFile') ? testFile : 'unknown'
|
|
task testSingle(dependsOn: [installDevTest]) {
|
|
doLast {
|
|
if (testFile != 'unknown') {
|
|
exec {
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && pytest ${testFile}"
|
|
}
|
|
} else {
|
|
throw new GradleException("No file provided. Use -PtestFile=<test_file>")
|
|
}
|
|
}
|
|
}
|
|
|
|
task testIntegrationBatch0(type: Exec, dependsOn: [installDevTest]) {
|
|
def cvg_arg = get_coverage_arg("intBatch0")
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"pytest ${cvg_arg} --durations=50 -m 'integration_batch_0' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch0.xml"
|
|
}
|
|
task testIntegrationBatch1(type: Exec, dependsOn: [installDevTest]) {
|
|
def cvg_arg = get_coverage_arg("intBatch1")
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"pytest ${cvg_arg} --durations=50 -m 'integration_batch_1' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch1.xml"
|
|
}
|
|
task testIntegrationBatch2(type: Exec, dependsOn: [installDevTest]) {
|
|
def cvg_arg = get_coverage_arg("intBatch2")
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"pytest ${cvg_arg} --durations=20 -m 'integration_batch_2' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch2.xml"
|
|
}
|
|
|
|
task testFull(type: Exec, dependsOn: [installDevTest]) {
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && set -x && " +
|
|
"pytest --durations=50 -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
|
|
}
|
|
|
|
task specGen(type: Exec, dependsOn: [codegen, installDevTest]) {
|
|
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/specgen.sh"
|
|
}
|
|
task docGen(type: Exec, dependsOn: [codegen, installDevTest, specGen]) {
|
|
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/docgen.sh"
|
|
}
|
|
|
|
|
|
|
|
task cleanPythonCache(type: Exec) {
|
|
commandLine 'bash', '-c',
|
|
"find src tests -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete"
|
|
}
|
|
task buildWheel(type: Exec, dependsOn: [install, codegen, cleanPythonCache]) {
|
|
commandLine 'bash', '-c',
|
|
"source ${venv_name}/bin/activate && " +
|
|
'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
|
|
}
|
|
|
|
build.dependsOn install
|
|
check.dependsOn lint
|
|
check.dependsOn testQuick
|
|
|
|
clean {
|
|
delete venv_name
|
|
delete 'build'
|
|
delete 'dist'
|
|
delete 'src/datahub/metadata'
|
|
delete '../docs/generated'
|
|
delete 'generated'
|
|
delete '.mypy_cache'
|
|
delete '.pytest_cache'
|
|
delete '.preflight_sentinel'
|
|
}
|
|
clean.dependsOn cleanPythonCache
|
|
|
|
idea {
|
|
module {
|
|
sourceDirs += file('src')
|
|
testSourceDirs += file('tests')
|
|
}
|
|
}
|