yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-03 12:16:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci(airflow): separate airflow constraints from deps (#13291)

Browse Source
This commit is contained in:
Harshal Sheth 2025-04-22 16:54:15 -04:00 committed by GitHub
parent 45c5a620e7
commit 82fafceba4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 19 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/airflow-plugin.yml vendored
View File

@ -35,17 +35,23 @@ jobs:
# Note: this should be kept in sync with tox.ini.
- python-version: "3.8"
extra_pip_extras: test-airflow25
# extra_pip_requirements: "apache-airflow~=2.5.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.5.3/constraints-3.8.txt"
extra_pip_requirements: "apache-airflow~=2.5.3"
# extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.5.3/constraints-3.8.txt"
- python-version: "3.10"
extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
extra_pip_requirements: "apache-airflow~=2.6.3"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
- python-version: "3.10"
extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
extra_pip_requirements: "apache-airflow~=2.7.3"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
- python-version: "3.10"
extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
extra_pip_requirements: "apache-airflow~=2.8.1"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
- python-version: "3.11"
extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
extra_pip_requirements: "apache-airflow~=2.9.3"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
- python-version: "3.11"
extra_pip_requirements: "apache-airflow~=2.10.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt"
extra_pip_requirements: "apache-airflow~=2.10.3"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt"
fail-fast: false
steps:
- name: Set up JDK 17
@ -62,7 +68,7 @@ jobs:
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }})
run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_constraints='${{ matrix.extra_pip_constraints }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
- name: pip freeze show list installed
if: always()
run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze

9
metadata-ingestion-modules/airflow-plugin/build.gradle
View File

@ -14,6 +14,9 @@ ext.venv_activate_command = "set +x && source ${venv_name}/bin/activate && set -
if (!project.hasProperty("extra_pip_requirements")) {
ext.extra_pip_requirements = ""
}
if (!project.hasProperty("extra_pip_constraints")) {
ext.extra_pip_constraints = ""
}
if (!project.hasProperty("extra_pip_extras")) {
ext.extra_pip_extras = ""
}
@ -22,7 +25,7 @@ if (extra_pip_extras != "") {
ext.extra_pip_extras = "," + extra_pip_extras
}
def pip_install_command = "VIRTUAL_ENV=${venv_name} ${venv_name}/bin/uv pip install -e ../../metadata-ingestion"
def pip_install_command = "VIRTUAL_ENV=${venv_name} ${venv_name}/bin/uv pip install -e ../../metadata-ingestion ${extra_pip_requirements}"
task environmentSetup(type: Exec) {
def sentinel_file = "${venv_name}/.venv_environment_sentinel"
@ -40,7 +43,7 @@ task installPackage(type: Exec, dependsOn: [environmentSetup, ':metadata-ingesti
outputs.file(sentinel_file)
commandLine 'bash', '-c',
venv_activate_command +
"${pip_install_command} -e .[ignore${extra_pip_extras}] ${extra_pip_requirements} &&" +
"${pip_install_command} -e .[ignore${extra_pip_extras}] ${extra_pip_constraints} &&" +
"touch ${sentinel_file}"
}
@ -63,7 +66,7 @@ task installTest(type: Exec, dependsOn: [installDev]) {
outputs.file(sentinel_file)
commandLine 'bash', '-c',
venv_activate_command +
"${pip_install_command} -e .[integration-tests${extra_pip_extras}] ${extra_pip_requirements} && " +
"${pip_install_command} -e .[integration-tests${extra_pip_extras}] ${extra_pip_constraints} && " +
"touch ${sentinel_file}"
}