/** * Copyright 2021 Acryl Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ plugins { id 'scala' id 'org.gradle.playframework' } apply from: "../gradle/versioning/versioning.gradle" apply from: "../gradle/coverage/python-coverage.gradle" apply from: '../gradle/docker/docker.gradle' ext { python_executable = 'python3' venv_name = 'venv' docker_registry = 'acryldata' docker_repo = 'datahub-actions' docker_target = project.getProperties().getOrDefault("dockerTarget", "slim") python_docker_version = project.getProperties().getOrDefault("pythonDockerVersion", "1!0.0.0+docker.${version}") } if (!project.hasProperty("extra_pip_requirements")) { ext.extra_pip_requirements = "" } def pip_install_command = "VIRTUAL_ENV=${venv_name} ${venv_name}/bin/uv pip install -e ../metadata-ingestion" task checkPythonVersion(type: Exec) { commandLine python_executable, '-c', 'import sys; sys.version_info >= (3, 8), f"Python version {sys.version_info[:2]} not allowed"' } task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { def sentinel_file = "${venv_name}/.venv_environment_sentinel" inputs.file file('setup.py') outputs.file(sentinel_file) commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && " + "${venv_name}/bin/python -m pip install --upgrade uv && " + "touch ${sentinel_file}" } task installPackage(type: Exec, dependsOn: [environmentSetup, ':metadata-ingestion:codegen']) { def sentinel_file = "${venv_name}/.build_install_package_sentinel" inputs.file file('setup.py') outputs.file(sentinel_file) commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "${pip_install_command} -e . ${extra_pip_requirements} && " + "touch ${sentinel_file}" } task install(dependsOn: [installPackage]) task installDev(type: Exec, dependsOn: [install]) { def sentinel_file = "${venv_name}/.build_install_dev_sentinel" inputs.file file('setup.py') outputs.file(sentinel_file) commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "${pip_install_command} -e .[dev] ${extra_pip_requirements} && " + "touch ${sentinel_file}" } task lint(type: Exec, dependsOn: installDev) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "ruff check src/ tests/ && " + "ruff format --check src/ tests/ && " + "mypy --show-traceback --show-error-codes src/ tests/" } task lintFix(type: Exec, dependsOn: installDev) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "ruff check --fix src/ tests/ && " + "ruff format src/ tests/ " } task installDevTest(type: Exec, dependsOn: [installDev]) { def sentinel_file = "${venv_name}/.build_install_dev_test_sentinel" inputs.file file('setup.py') outputs.dir("${venv_name}") outputs.file(sentinel_file) commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "${pip_install_command} -e .[dev,integration-tests] ${extra_pip_requirements} && " + "touch ${sentinel_file}" } task testQuick(type: Exec, dependsOn: installDevTest) { // We can't enforce the coverage requirements if we run a subset of the tests. inputs.files(project.fileTree(dir: "src/", include: "**/*.py")) inputs.files(project.fileTree(dir: "tests/")) outputs.dir("${venv_name}") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "pytest -vv ${get_coverage_args('quick')} --continue-on-collection-errors --junit-xml=junit.quick.xml" } def testFile = hasProperty('testFile') ? testFile : 'unknown' task testSingle(dependsOn: [installDevTest]) { doLast { if (testFile != 'unknown') { exec { commandLine 'bash', '-x', '-c', "source ${venv_name}/bin/activate && pytest ${testFile}" } } else { throw new GradleException("No file provided. Use -PtestFile=") } } } task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) { commandLine 'bash', '-x', '-c', "source ${venv_name}/bin/activate && pytest -vv ${get_coverage_args('full')} --continue-on-collection-errors --junit-xml=junit.full.xml" } task buildWheel(type: Exec, dependsOn: [environmentSetup]) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh' } task cleanPythonCache(type: Exec) { commandLine 'bash', '-x', '-c', "find src -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete" } docker { //dependsOn(build) name "${docker_registry}/${docker_repo}:${versionTag}" dockerfile file("${rootProject.projectDir}/docker/datahub-actions/Dockerfile") files fileTree(rootProject.projectDir) { exclude "datahub-actions/scripts/**" exclude "datahub-actions/build/**" exclude "datahub-actions/venv/**" exclude "datahub-actions/tests/**" exclude "**/*.xml" include ".dockerignore" include "docker/datahub-actions/**" include "docker/snippets/**" include "datahub-actions/**" include "python-build/**" }.exclude { i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden()) } additionalTag("Debug", "${docker_registry}/${docker_repo}:debug") defaultVariant = "slim" variants = [ "slim": [suffix: "-slim", args: [APP_ENV: "slim", RELEASE_VERSION: python_docker_version]], "full": [suffix: "", args: [APP_ENV: "full", RELEASE_VERSION: python_docker_version]] ] } build.dependsOn install check.dependsOn lint check.dependsOn testQuick clean { delete venv_name delete 'build' delete 'dist' } clean.dependsOn cleanPythonCache