yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-18 14:16:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci(ingest): make ingestion caching rules correct (#8685)

Browse Source
This commit is contained in:
Harshal Sheth 2023-08-23 22:06:28 -07:00 committed by GitHub
parent a97548ce46
commit 090f8af8e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
metadata-ingestion/build.gradle
View File

@ -21,11 +21,13 @@ task checkPythonVersion(type: Exec) {
} }
task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
def sentinel_file = "${venv_name}/.venv_environment_sentinel"
inputs.file file('setup.py') inputs.file file('setup.py')
outputs.dir("${venv_name}") outputs.file(sentinel_file)
commandLine 'bash', '-c', commandLine 'bash', '-c',
"${python_executable} -m venv ${venv_name} && " + "${python_executable} -m venv ${venv_name} && " +
"${venv_name}/bin/python -m pip install --upgrade pip wheel 'setuptools>=63.0.0'" "${venv_name}/bin/python -m pip install --upgrade pip wheel 'setuptools>=63.0.0' && " +
"touch ${sentinel_file}"
} }
task runPreFlightScript(type: Exec, dependsOn: environmentSetup) { task runPreFlightScript(type: Exec, dependsOn: environmentSetup) {
@ -39,7 +41,6 @@ task runPreFlightScript(type: Exec, dependsOn: environmentSetup) {
task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) { task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) {
def sentinel_file = "${venv_name}/.build_install_package_only_sentinel" def sentinel_file = "${venv_name}/.build_install_package_only_sentinel"
inputs.file file('setup.py') inputs.file file('setup.py')
outputs.dir("${venv_name}")
outputs.file(sentinel_file) outputs.file(sentinel_file)
commandLine 'bash', '-x', '-c', commandLine 'bash', '-x', '-c',
"${venv_name}/bin/pip install -e . &&" + "${venv_name}/bin/pip install -e . &&" +
@ -47,9 +48,12 @@ task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) {
} }
task installPackage(type: Exec, dependsOn: installPackageOnly) { task installPackage(type: Exec, dependsOn: installPackageOnly) {
def sentinel_file = "${venv_name}/.build_install_package_sentinel"
inputs.file file('setup.py') inputs.file file('setup.py')
outputs.dir("${venv_name}") outputs.file(sentinel_file)
commandLine 'bash', '-x', '-c', "${venv_name}/bin/pip install -e . ${extra_pip_requirements}" commandLine 'bash', '-x', '-c',
"${venv_name}/bin/pip install -e . ${extra_pip_requirements} && " +
"touch ${sentinel_file}"
} }
task codegen(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadata-events:mxe-schemas:build']) { task codegen(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadata-events:mxe-schemas:build']) {
@ -63,7 +67,6 @@ task install(dependsOn: [installPackage, codegen])
task installDev(type: Exec, dependsOn: [install]) { task installDev(type: Exec, dependsOn: [install]) {
def sentinel_file = "${venv_name}/.build_install_dev_sentinel" def sentinel_file = "${venv_name}/.build_install_dev_sentinel"
inputs.file file('setup.py') inputs.file file('setup.py')
outputs.dir("${venv_name}")
outputs.file(sentinel_file) outputs.file(sentinel_file)
commandLine 'bash', '-c', commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " + "source ${venv_name}/bin/activate && set -x && " +
@ -75,7 +78,6 @@ task installDev(type: Exec, dependsOn: [install]) {
task installAll(type: Exec, dependsOn: [install]) { task installAll(type: Exec, dependsOn: [install]) {
def sentinel_file = "${venv_name}/.build_install_all_sentinel" def sentinel_file = "${venv_name}/.build_install_all_sentinel"
inputs.file file('setup.py') inputs.file file('setup.py')
outputs.dir("${venv_name}")
outputs.file(sentinel_file) outputs.file(sentinel_file)
commandLine 'bash', '-c', commandLine 'bash', '-c',
"source ${venv_name}/bin/activate && set -x && " + "source ${venv_name}/bin/activate && set -x && " +